diff --git a/scripts/ci/extends/test-artifacts.yml b/scripts/ci/extends/test-artifacts.yml
index 27700422e2aef0826e690b58f723cf4a757e912b..cf865c4e6ec9258d765b671600c625ac9d3cdf3e 100644
--- a/scripts/ci/extends/test-artifacts.yml
+++ b/scripts/ci/extends/test-artifacts.yml
@@ -2,7 +2,7 @@
artifacts:
when: always
paths:
- - build/*/logs/*.txt
+ - build/*/logs
- build/*/Tests/ctest-junit.xml
- build/*/Tests/testrunner.xml
- build/*/make.txt
diff --git a/scripts/cmake/PythonSetup.cmake b/scripts/cmake/PythonSetup.cmake
index 26efb4067e046a99fdc620429e3f3fd23b4f678a..329e35ce5463f91cc297ba44aa90dbadba9cc5f2 100644
--- a/scripts/cmake/PythonSetup.cmake
+++ b/scripts/cmake/PythonSetup.cmake
@@ -210,9 +210,7 @@ function(setup_venv_dependent_ctests)
EXECUTABLE ogs
EXECUTABLE_ARGS ${ht_invalid_prj_file}
RUNTIME 1
- )
- set_tests_properties(
- ogs-HT_${ht_invalid_prj_file_short} PROPERTIES WILL_FAIL TRUE
+ PROPERTIES WILL_FAIL TRUE
)
endforeach()
endif()
diff --git a/scripts/cmake/test/AddTest.cmake b/scripts/cmake/test/AddTest.cmake
index 6e84772c2634c7b9e27e8408e5d7fb6dbf26a56a..9d35517e9e2df5e9713f257d2269ced6895ceb53 100644
--- a/scripts/cmake/test/AddTest.cmake
+++ b/scripts/cmake/test/AddTest.cmake
@@ -346,6 +346,9 @@ macro(_add_test TEST_NAME)
)
endif()
+ isTestCommandExpectedToSucceed(${TEST_NAME} ${AddTest_PROPERTIES})
+ message(DEBUG "Is test '${TEST_NAME}' expected to succeed? → ${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}")
+
add_test(
NAME ${TEST_NAME}
COMMAND
@@ -355,8 +358,10 @@ macro(_add_test TEST_NAME)
-DBINARY_PATH=${_binary_path} -DWRAPPER_COMMAND=${WRAPPER_COMMAND}
"-DWRAPPER_ARGS=${AddTest_WRAPPER_ARGS}"
-DWORKING_DIRECTORY=${AddTest_WORKING_DIRECTORY}
- -DLOG_FILE=${PROJECT_BINARY_DIR}/logs/${TEST_NAME}.txt -P
- ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
+ "-DLOG_ROOT=${PROJECT_BINARY_DIR}/logs"
+ "-DLOG_FILE_BASENAME=${TEST_NAME}.txt"
+ "-DTEST_COMMAND_IS_EXPECTED_TO_SUCCEED=${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}"
+ -P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
)
if(DEFINED AddTest_DEPENDS)
@@ -607,3 +612,47 @@ Use six arguments version of AddTest with absolute and relative tolerances"
${AddTest_DISABLED} LABELS "tester;${labels}"
)
endmacro()
+
+# Checks if a test is expected to succeed based on the properties WILL_FAIL,
+# PASS_REGULAR_EXPRESSION and FAIL_REGULAR_EXPRESSION.
+# The function expects the test name (used only for debugging purposes) and the
+# test properties as arguments.
+# The test does not need to exist, yet. This function does not query any test
+# case, but only uses the passed list of properties
+function(isTestCommandExpectedToSucceed TEST_NAME)
+ set(options WILL_FAIL)
+ set(oneValueArgs PASS_REGULAR_EXPRESSION FAIL_REGULAR_EXPRESSION)
+ set(multiValueArgs)
+ cmake_parse_arguments(TEST_FAILURE "${options}" "${oneValueArgs}"
+ "${multiValueArgs}" ${ARGN})
+
+ message(DEBUG "failure properties for test ${TEST_NAME}:")
+ list(APPEND CMAKE_MESSAGE_INDENT " ")
+ message(DEBUG "WILL_FAIL: ${TEST_FAILURE_WILL_FAIL}")
+ message(DEBUG "PASS_RE: ${TEST_FAILURE_PASS_REGULAR_EXPRESSION}")
+ message(DEBUG "FAIL_RE: ${TEST_FAILURE_FAIL_REGULAR_EXPRESSION}")
+ list(POP_BACK CMAKE_MESSAGE_INDENT)
+
+ if (${TEST_FAILURE_WILL_FAIL})
+ if (DEFINED TEST_FAILURE_PASS_REGULAR_EXPRESSION)
+ # Note: if the test property PASS_REGULAR_EXPRESSION is set, the
+ # process return code will be ignored, see https://cmake.org/cmake/help/latest/prop_test/PASS_REGULAR_EXPRESSION.html
+ message(SEND_ERROR "Error in test '${TEST_NAME}': Please do not use both WILL_FAIL and PASS_REGULAR_EXPRESSION in the same test. The logic will be unclear, then.")
+ endif()
+ if (DEFINED TEST_FAILURE_FAIL_REGULAR_EXPRESSION)
+ message(SEND_ERROR "Error in test '${TEST_NAME}': Please do not use both WILL_FAIL and FAIL_REGULAR_EXPRESSION in the same test. The logic will be unclear, then.")
+ endif()
+
+ set(TEST_COMMAND_IS_EXPECTED_TO_SUCCEED false)
+ elseif(DEFINED TEST_FAILURE_PASS_REGULAR_EXPRESSION)
+ if (DEFINED TEST_FAILURE_FAIL_REGULAR_EXPRESSION)
+ message(SEND_ERROR "Error in test '${TEST_NAME}': Please do not use both PASS_REGULAR_EXPRESSION and FAIL_REGULAR_EXPRESSION in the same test. The logic will be unclear, then.")
+ endif()
+
+ set(TEST_COMMAND_IS_EXPECTED_TO_SUCCEED false)
+ else()
+ set(TEST_COMMAND_IS_EXPECTED_TO_SUCCEED true)
+ endif()
+
+ set(TEST_COMMAND_IS_EXPECTED_TO_SUCCEED "${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}" PARENT_SCOPE)
+endfunction()
diff --git a/scripts/cmake/test/AddTestWrapper.cmake b/scripts/cmake/test/AddTestWrapper.cmake
index c9a3116a010609dcfdf6dfe05a19f43445b4676b..2249f7b7e81f56f483bb6aed35c6a10dfe1822d1 100644
--- a/scripts/cmake/test/AddTestWrapper.cmake
+++ b/scripts/cmake/test/AddTestWrapper.cmake
@@ -1,4 +1,4 @@
-# IMPORTANT: multiple arguments in one variables have to be in list notation (;)
+# IMPORTANT: multiple arguments in a single variable have to be in list notation (;)
# and have to be quoted when passed
# "-DEXECUTABLE_ARGS=${AddTest_EXECUTABLE_ARGS}"
execute_process(
@@ -12,9 +12,43 @@ execute_process(
COMMAND_ECHO STDOUT
)
-if(EXIT_CODE STREQUAL "0" AND NOT DEFINED ENV{CI})
- file(WRITE ${LOG_FILE} "${LOG}")
-elseif(NOT EXIT_CODE STREQUAL "0")
- file(WRITE ${LOG_FILE} "${LOG}")
+set(SAVE_LOG true)
+set(TEST_LOG_DIR "${LOG_ROOT}")
+
+if (TEST_COMMAND_IS_EXPECTED_TO_SUCCEED)
+ if (EXIT_CODE STREQUAL "0")
+ # expected: success, actual: success
+ if (DEFINED ENV{CI})
+ set(SAVE_LOG false)
+ endif()
+ else()
+ # expected: success, actual: failure
+ # use default settings
+ endif()
+else()
+ if (EXIT_CODE STREQUAL "0")
+ # expected: failure, actual: success
+ if (DEFINED ENV{CI})
+ set(TEST_LOG_DIR "${LOG_ROOT}/command_succeeded_but_was_expected_to_fail")
+ endif()
+ else()
+ # expected: failure, actual: failure
+ if (DEFINED ENV{CI})
+ set(TEST_LOG_DIR "${LOG_ROOT}/command_failed_as_expected")
+ endif()
+ endif()
+endif()
+
+set(LOG_FILE "${TEST_LOG_DIR}/${LOG_FILE_BASENAME}")
+
+if (SAVE_LOG)
+ if(NOT EXISTS "${TEST_LOG_DIR}")
+ file(MAKE_DIRECTORY "${TEST_LOG_DIR}")
+ endif()
+
+ file(WRITE "${LOG_FILE}" "${LOG}")
+endif()
+
+if(NOT EXIT_CODE STREQUAL "0")
message(FATAL_ERROR "Exit code: ${EXIT_CODE}; log file: ${LOG_FILE}")
endif()
diff --git a/scripts/cmake/test/NotebookTest.cmake b/scripts/cmake/test/NotebookTest.cmake
index 1d2740d80c1c7daefd8907e30a90f28378db4d55..82e6d91475132a2877877fcbf1acf02bfd8ad615 100644
--- a/scripts/cmake/test/NotebookTest.cmake
+++ b/scripts/cmake/test/NotebookTest.cmake
@@ -98,6 +98,9 @@ function(NotebookTest)
endif()
endif()
+ isTestCommandExpectedToSucceed(${TEST_NAME} ${NotebookTest_PROPERTIES})
+ message(DEBUG "Is test '${TEST_NAME}' expected to succeed? → ${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}")
+
add_test(
NAME ${TEST_NAME}
COMMAND
@@ -105,7 +108,9 @@ function(NotebookTest)
# TODO: only works if notebook is in a leaf directory
-DEXECUTABLE=${Python_EXECUTABLE} "-DEXECUTABLE_ARGS=${_exe_args}"
-DWORKING_DIRECTORY=${Data_SOURCE_DIR}
- -DLOG_FILE=${PROJECT_BINARY_DIR}/logs/${NotebookTest_NAME_WE}.txt
+ "-DLOG_ROOT=${PROJECT_BINARY_DIR}/logs"
+ "-DLOG_FILE_BASENAME=${NotebookTest_NAME_WE}.txt"
+ "-DTEST_COMMAND_IS_EXPECTED_TO_SUCCEED=${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}"
-P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
)
diff --git a/scripts/cmake/test/OgsTest.cmake b/scripts/cmake/test/OgsTest.cmake
index 360b4178fcee78e0344a690c64e815a24e256095..900bce228745c182758e997f049ba73bd179a3e4 100644
--- a/scripts/cmake/test/OgsTest.cmake
+++ b/scripts/cmake/test/OgsTest.cmake
@@ -90,7 +90,7 @@ function(OgsTest)
endif()
endfunction()
-# Add a ctest and sets properties
+# Adds a ctest and sets properties
macro(_ogs_add_test TEST_NAME)
if("${TEST_NAME}" MATCHES "-omp")
set(OgsTest_BINARY_DIR "${Data_BINARY_DIR}/${OgsTest_DIR}-omp")
@@ -100,6 +100,10 @@ macro(_ogs_add_test TEST_NAME)
file(MAKE_DIRECTORY ${OgsTest_BINARY_DIR})
file(TO_NATIVE_PATH "${OgsTest_BINARY_DIR}" OgsTest_BINARY_DIR_NATIVE)
string(REPLACE "/" "_" TEST_NAME_UNDERSCORE ${TEST_NAME})
+
+ isTestCommandExpectedToSucceed(${TEST_NAME} ${OgsTest_PROPERTIES})
+ message(DEBUG "Is test '${TEST_NAME}' expected to succeed? → ${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}")
+
add_test(
NAME ${TEST_NAME}
COMMAND
@@ -107,7 +111,9 @@ macro(_ogs_add_test TEST_NAME)
"-DEXECUTABLE_ARGS=${_exe_args}"
"-DWRAPPER_COMMAND=${OgsTest_WRAPPER}"
-DWORKING_DIRECTORY=${OgsTest_BINARY_DIR}
- -DLOG_FILE=${PROJECT_BINARY_DIR}/logs/${TEST_NAME_UNDERSCORE}.txt
+ "-DLOG_FILE_BASENAME=${TEST_NAME_UNDERSCORE}.txt"
+ "-DLOG_ROOT=${PROJECT_BINARY_DIR}/logs"
+ "-DTEST_COMMAND_IS_EXPECTED_TO_SUCCEED=${TEST_COMMAND_IS_EXPECTED_TO_SUCCEED}"
-P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
)