From 7d079f9c72203fb08db34baab548f9cd7104355d Mon Sep 17 00:00:00 2001
From: Lars Bilke <lars.bilke@ufz.de>
Date: Mon, 7 Apr 2014 16:25:27 +0200
Subject: [PATCH] First testing environment implementation.
See scripts/cmake/test/README.md for details.
Conflicts:
SimpleTests/FemTests/CMakeLists.txt (ignored)
---
CMakeLists.txt | 2 +-
scripts/cmake/Test.cmake | 0
scripts/cmake/test/AddTest.cmake | 50 +++++++++++++++++++++++++
scripts/cmake/test/AddTestTester.cmake | 15 ++++++++
scripts/cmake/test/AddTestWrapper.cmake | 12 ++++++
scripts/cmake/test/CTestCustom.cmake.in | 2 +
scripts/cmake/test/README.md | 17 +++++++++
scripts/cmake/test/Test.cmake | 18 +++++++++
8 files changed, 115 insertions(+), 1 deletion(-)
delete mode 100644 scripts/cmake/Test.cmake
create mode 100644 scripts/cmake/test/AddTest.cmake
create mode 100644 scripts/cmake/test/AddTestTester.cmake
create mode 100644 scripts/cmake/test/AddTestWrapper.cmake
create mode 100644 scripts/cmake/test/CTestCustom.cmake.in
create mode 100644 scripts/cmake/test/README.md
create mode 100644 scripts/cmake/test/Test.cmake
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1cb91d27654..25788e7ee70 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ INCLUDE(scripts/cmake/Find.cmake)
INCLUDE(scripts/cmake/SubmoduleSetup.cmake)
INCLUDE(scripts/cmake/ProjectSetup.cmake)
INCLUDE(scripts/cmake/DocumentationSetup.cmake)
-INCLUDE(scripts/cmake/Test.cmake)
+INCLUDE(scripts/cmake/test/Test.cmake)
IF(OGS_COVERAGE AND NOT IS_SUBPROJECT)
INCLUDE(scripts/cmake/Coverage.cmake)
ENDIF()
diff --git a/scripts/cmake/Test.cmake b/scripts/cmake/Test.cmake
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/scripts/cmake/test/AddTest.cmake b/scripts/cmake/test/AddTest.cmake
new file mode 100644
index 00000000000..f0f1e82a48f
--- /dev/null
+++ b/scripts/cmake/test/AddTest.cmake
@@ -0,0 +1,50 @@
+FUNCTION (AddTest executable case_path case_name wrapper)
+
+ SET(tester ${ARGV4})
+
+ # Implement wrappers
+ IF(wrapper STREQUAL "TIME")
+ SET(WRAPPER_COMMAND ${TIME_TOOL_PATH})
+ ELSEIF(wrapper STREQUAL "MEMCHECK" AND VALGRIND_TOOL_PATH)
+ SET(WRAPPER_COMMAND "${VALGRIND_TOOL_PATH} --tool=memcheck --log-file=${case_path}/${case_name}_memcheck.log -v --leak-check=full --show-reachable=yes --track-origins=yes --malloc-fill=0xff --free-fill=0xff")
+ SET(tester MEMCHECK)
+ ELSEIF(wrapper STREQUAL "CALLGRIND" AND VALGRIND_TOOL_PATH)
+ SET(WRAPPER_COMMAND "${VALGRIND_TOOL_PATH} --tool=callgrind --branch-sim=yes --cache-sim=yes --dump-instr=yes --collect-jumps=yes")
+ UNSET(tester)
+ ENDIF()
+
+ # Implement testers
+ IF(tester STREQUAL "DIFF")
+ SET(TESTER_COMMAND "${DIFF_TOOL_PATH} -sbB ${case_path}/${case_name}_expected_result.vtu ${case_path}/${case_name}_with_results.vtu")
+ ELSEIF(tester STREQUAL "MEMCHECK")
+ SET(TESTER_COMMAND "! ${GREP_TOOL_PATH} definitely ${case_path}/${case_name}_memcheck.log")
+ ENDIF()
+
+ ## -----------
+
+ ADD_TEST(NAME "${executable}-${case_path}-${wrapper}"
+ COMMAND ${CMAKE_COMMAND}
+ -Dexecutable=$<TARGET_FILE:${executable}>
+ -Dcase_path=${case_path}
+ -Dcase_name=${case_name}
+ -Dwrapper=${wrapper}
+ -DWRAPPER_COMMAND=${WRAPPER_COMMAND}
+ -DCMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}
+ -P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
+ )
+
+ IF(NOT tester)
+ RETURN()
+ ENDIF()
+
+ ADD_TEST(NAME "${executable}-${case_path}-${wrapper}-${tester}"
+ COMMAND ${CMAKE_COMMAND}
+ -Dcase_path=${case_path}
+ -Dcase_name=${case_name}
+ -Dtester=${tester}
+ -DTESTER_COMMAND=${TESTER_COMMAND}
+ -DCMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}
+ -P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestTester.cmake
+ )
+
+ENDFUNCTION()
diff --git a/scripts/cmake/test/AddTestTester.cmake b/scripts/cmake/test/AddTestTester.cmake
new file mode 100644
index 00000000000..d0f30840c72
--- /dev/null
+++ b/scripts/cmake/test/AddTestTester.cmake
@@ -0,0 +1,15 @@
+#message("tester: ${TESTER_COMMAND}")
+#STRING(REPLACE " " ";" TESTER_COMMAND ${TESTER_COMMAND})
+#set(list ${TESTER_COMMAND})
+#message("tester: ${list}")
+
+
+EXECUTE_PROCESS(
+ COMMAND bash -c ${TESTER_COMMAND}
+ WORKING_DIRECTORY ${case_path}
+ RESULT_VARIABLE EXIT_CODE
+)
+
+IF(NOT EXIT_CODE STREQUAL "0")
+ MESSAGE(FATAL_ERROR "Error exit code: ${EXIT_CODE}")
+ENDIF()
diff --git a/scripts/cmake/test/AddTestWrapper.cmake b/scripts/cmake/test/AddTestWrapper.cmake
new file mode 100644
index 00000000000..0b900a6fdae
--- /dev/null
+++ b/scripts/cmake/test/AddTestWrapper.cmake
@@ -0,0 +1,12 @@
+SET(ARGS --boundary_condition ${case_path}/${case_name}.cnd -g ${case_path}/${case_name}.gml -m ${case_path}/${case_name}.vtu)
+
+STRING(REPLACE " " ";" WRAPPER_COMMAND ${WRAPPER_COMMAND})
+EXECUTE_PROCESS(
+ COMMAND ${WRAPPER_COMMAND} ${executable} ${ARGS}
+ WORKING_DIRECTORY ${case_path}
+ RESULT_VARIABLE EXIT_CODE
+)
+
+IF(NOT EXIT_CODE STREQUAL "0")
+ MESSAGE(FATAL_ERROR "Test wrapper exited with code: ${EXIT_CODE}")
+ENDIF()
diff --git a/scripts/cmake/test/CTestCustom.cmake.in b/scripts/cmake/test/CTestCustom.cmake.in
new file mode 100644
index 00000000000..66887f9e0f6
--- /dev/null
+++ b/scripts/cmake/test/CTestCustom.cmake.in
@@ -0,0 +1,2 @@
+# SET(CTEST_CUSTOM_POST_TEST "cat ${CMAKE_BINARY_DIR}/Testing/Temporary/LastTestTester.log 2>&1")
+SET(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE}) # logog
diff --git a/scripts/cmake/test/README.md b/scripts/cmake/test/README.md
new file mode 100644
index 00000000000..b7c75291c21
--- /dev/null
+++ b/scripts/cmake/test/README.md
@@ -0,0 +1,17 @@
+Required tools:
+
+- `time`
+- `diff`
+- `valgrind`
+- `grep`
+- `bash`
+
+Test data is searched for in `ogs6-sources/../ogs6-data`.
+
+In the build directory run `ctest` with `–-output-on-failure` and `-R` for includes and `-E` for excludes:
+
+```bash
+ctest --output-on-failure -R "MEMCHECK|VALGRIND" -E DIFF
+```
+
+Wrapper and tester are implemented in `AddTest.cmake`.
diff --git a/scripts/cmake/test/Test.cmake b/scripts/cmake/test/Test.cmake
new file mode 100644
index 00000000000..132d9843d8b
--- /dev/null
+++ b/scripts/cmake/test/Test.cmake
@@ -0,0 +1,18 @@
+# Find tools and data
+FIND_PROGRAM(TIME_TOOL_PATH time REQUIRED)
+FIND_PROGRAM(DIFF_TOOL_PATH diff REQUIRED)
+FIND_PROGRAM(GREP_TOOL_PATH grep REQUIRED)
+FIND_PROGRAM(BASH_TOOL_PATH bash REQUIRED)
+FIND_PROGRAM(VALGRIND_TOOL_PATH valgrind REQUIRED)
+FIND_FILE(OGS-DATA_PATH ".ogs6-data.dummy" REQUIRED
+ HINTS ${CMAKE_SOURCE_DIR}/../ogs6-data)
+GET_FILENAME_COMPONENT(OGS-DATA_PATH ${OGS-DATA_PATH} PATH)
+
+ENABLE_TESTING() # Enable CTest
+
+# See http://www.vtk.org/Wiki/CMake/Testing_With_CTest for some customization options
+SET(CTEST_CUSTOM_TESTS_IGNORE test-harness) # ignore logog test
+CONFIGURE_FILE(
+ ${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/CTestCustom.cmake.in
+ ${CMAKE_BINARY_DIR}/CTestCustom.cmake
+)
--
GitLab