From 9e89dfcbe7a6ac22c8163af5704c237c7e3d3c0c Mon Sep 17 00:00:00 2001
From: Lars Bilke <lars.bilke@ufz.de>
Date: Wed, 12 Jan 2022 22:02:02 +0100
Subject: [PATCH] [T] Added NotebookTest() as CMake integration for notebook
testrunner.py.
---
CMakeLists.txt | 1 +
.../Notebooks/FailingNotebook.ci-skip.ipynb | 33 +++++++
scripts/ci/extends/test-artifacts.yml | 3 +
scripts/ci/jobs/build-linux.yml | 1 +
scripts/cmake/test/NotebookTest.cmake | 89 +++++++++++++++++++
scripts/cmake/test/Test.cmake | 7 ++
6 files changed, 134 insertions(+)
create mode 100644 Tests/Data/Notebooks/FailingNotebook.ci-skip.ipynb
create mode 100644 scripts/cmake/test/NotebookTest.cmake
diff --git a/CMakeLists.txt b/CMakeLists.txt
index f5826bedc3b..a80aff1e19d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -55,6 +55,7 @@ set(OGS_PETSC_CONFIG_OPTIONS "" CACHE STRING
option(OGS_USE_CVODE "Use the Sundials CVODE module?" OFF)
option(OGS_BUILD_UTILS "Should the utilities programs be built?" ON)
option(OGS_BUILD_TESTING "Should the tests be built?" ON)
+cmake_dependent_option(OGS_TEST_NOTEBOOKS "Should notebooks be tested?" ON "OGS_BUILD_TESTING" OFF)
if(MSVC)
set(CMD_COMMAND "cmd;/c")
diff --git a/Tests/Data/Notebooks/FailingNotebook.ci-skip.ipynb b/Tests/Data/Notebooks/FailingNotebook.ci-skip.ipynb
new file mode 100644
index 00000000000..8ae8c9b1a65
--- /dev/null
+++ b/Tests/Data/Notebooks/FailingNotebook.ci-skip.ipynb
@@ -0,0 +1,33 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Test Notebook assertion for testrunner\n",
+ "\n",
+ "A notebook test will fail with `assert False` or `raise SystemExit()`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "assert False\n",
+ "\n",
+ "# This line will not be reached but would also work:\n",
+ "raise SystemExit()"
+ ]
+ }
+ ],
+ "metadata": {
+ "language_info": {
+ "name": "python"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/scripts/ci/extends/test-artifacts.yml b/scripts/ci/extends/test-artifacts.yml
index 3f750095af1..048563291c1 100644
--- a/scripts/ci/extends/test-artifacts.yml
+++ b/scripts/ci/extends/test-artifacts.yml
@@ -8,6 +8,9 @@
- build/*/make.txt
- build/*/*.zip
- build/*/*.tar.gz
+ # Notebook testrunner:
+ - build/**/*.ipynb.html
+ - build/**/out.txt
expire_in: 1 week
reports:
junit:
diff --git a/scripts/ci/jobs/build-linux.yml b/scripts/ci/jobs/build-linux.yml
index 24ebb7be38d..d4af26b8e28 100644
--- a/scripts/ci/jobs/build-linux.yml
+++ b/scripts/ci/jobs/build-linux.yml
@@ -70,6 +70,7 @@ build linux arch:
CMAKE_ARGS: >-
-DBUILD_SHARED_LIBS=ON
-DOGS_USE_POETRY=OFF
+ -DOGS_TEST_NOTEBOOKS=OFF
build linux ubuntu:
extends:
diff --git a/scripts/cmake/test/NotebookTest.cmake b/scripts/cmake/test/NotebookTest.cmake
new file mode 100644
index 00000000000..c2e2d51b139
--- /dev/null
+++ b/scripts/cmake/test/NotebookTest.cmake
@@ -0,0 +1,89 @@
+# cmake-lint: disable=C0103
+function(NotebookTest)
+
+ if(NOT OGS_BUILD_CLI OR NOT OGS_BUILD_TESTING OR NOT OGS_TEST_NOTEBOOKS)
+ return()
+ endif()
+ set(options DISABLED)
+ set(oneValueArgs NOTEBOOKFILE RUNTIME)
+ set(multiValueArgs WRAPPER)
+ cmake_parse_arguments(
+ NotebookTest "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}
+ )
+
+ get_filename_component(
+ NotebookTest_DIR "${NotebookTest_NOTEBOOKFILE}" DIRECTORY
+ )
+ get_filename_component(
+ NotebookTest_NAME "${NotebookTest_NOTEBOOKFILE}" NAME
+ )
+ get_filename_component(
+ NotebookTest_NAME_WE "${NotebookTest_NOTEBOOKFILE}" NAME_WE
+ )
+
+ if(NotebookTest_UNPARSED_ARGUMENTS)
+ message(
+ FATAL_ERROR
+ "Unparsed argument(s) '${NotebookTest_UNPARSED_ARGUMENTS}' to NotebookTest call."
+ )
+ endif()
+
+ set(timeout ${ogs.ctest.large_runtime})
+ if(DEFINED NotebookTest_RUNTIME)
+ math(EXPR timeout "${NotebookTest_RUNTIME} * 3")
+ else()
+ set(NotebookTest_RUNTIME 1)
+ endif()
+
+ if(DEFINED OGS_CTEST_MAX_RUNTIME)
+ if(${NotebookTest_RUNTIME} GREATER ${OGS_CTEST_MAX_RUNTIME})
+ return()
+ endif()
+ endif()
+ if(${NotebookTest_RUNTIME} GREATER ${ogs.ctest.large_runtime})
+ string(APPEND NotebookTest_NAME_WE "-LARGE")
+ endif()
+
+ set(NotebookTest_SOURCE_DIR "${Data_SOURCE_DIR}/${NotebookTest_DIR}")
+ set(NotebookTest_BINARY_DIR "${Data_BINARY_DIR}/${NotebookTest_DIR}")
+ file(MAKE_DIRECTORY ${NotebookTest_BINARY_DIR})
+ file(TO_NATIVE_PATH "${NotebookTest_BINARY_DIR}"
+ NotebookTest_BINARY_DIR_NATIVE
+ )
+
+ set(TEST_NAME "nb-${NotebookTest_DIR}/${NotebookTest_NAME_WE}")
+
+ set(_exe_args Notebooks/testrunner.py --out ${Data_BINARY_DIR}
+ ${NotebookTest_SOURCE_DIR}/${NotebookTest_NAME}
+ )
+ string(REPLACE "/" "_" TEST_NAME_UNDERSCORE ${TEST_NAME})
+ add_test(
+ NAME ${TEST_NAME}
+ COMMAND
+ ${CMAKE_COMMAND} -DEXECUTABLE=${Python3_EXECUTABLE}
+ "-DEXECUTABLE_ARGS=${_exe_args}"
+ -DWORKING_DIRECTORY=${Data_SOURCE_DIR} -DCAT_LOG=TRUE -P
+ ${PROJECT_SOURCE_DIR}/scripts/cmake/test/OgsTestWrapper.cmake
+ )
+
+ current_dir_as_list(ProcessLib labels)
+ list(APPEND labels Notebook)
+ if(${NotebookTest_RUNTIME} LESS_EQUAL ${ogs.ctest.large_runtime})
+ list(APPEND labels default)
+ else()
+ list(APPEND labels large)
+ endif()
+
+ set_tests_properties(
+ ${TEST_NAME}
+ PROPERTIES ENVIRONMENT
+ PATH=$<TARGET_FILE_DIR:ogs>:$ENV{PATH}
+ COST
+ ${NotebookTest_RUNTIME}
+ DISABLED
+ ${NotebookTest_DISABLED}
+ LABELS
+ "${labels}"
+ )
+
+endfunction()
diff --git a/scripts/cmake/test/Test.cmake b/scripts/cmake/test/Test.cmake
index 26de225a91c..ccd98eb0284 100644
--- a/scripts/cmake/test/Test.cmake
+++ b/scripts/cmake/test/Test.cmake
@@ -48,6 +48,13 @@ enable_testing() # Enable CTest
include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/AddTest.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/MeshTest.cmake)
include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/OgsTest.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/NotebookTest.cmake)
+
+# Check notebook testrunner
+NotebookTest(NOTEBOOKFILE Notebooks/FailingNotebook.ci-skip.ipynb RUNTIME 2)
+if(TEST nb-Notebooks/FailingNotebook)
+ set_tests_properties(nb-Notebooks/FailingNotebook PROPERTIES WILL_FAIL TRUE)
+endif()
if(CMAKE_CONFIGURATION_TYPES)
set(CONFIG_PARAMETER --build-config "$<CONFIGURATION>")
--
GitLab