First testing environment implementation.

See scripts/cmake/test/README.md for details.

Conflicts:
	SimpleTests/FemTests/CMakeLists.txt (ignored)

CMakeLists.txt

@@ -37,7 +37,7 @@ INCLUDE(scripts/cmake/Find.cmake)
 INCLUDE(scripts/cmake/SubmoduleSetup.cmake)
 INCLUDE(scripts/cmake/ProjectSetup.cmake)
 INCLUDE(scripts/cmake/DocumentationSetup.cmake)
-INCLUDE(scripts/cmake/Test.cmake)
+INCLUDE(scripts/cmake/test/Test.cmake)
 IF(OGS_COVERAGE AND NOT IS_SUBPROJECT)
 	INCLUDE(scripts/cmake/Coverage.cmake)
 ENDIF()

scripts/cmake/test/AddTest.cmake

+FUNCTION (AddTest executable case_path case_name wrapper)
+
+	SET(tester ${ARGV4})
+
+	# Implement wrappers
+	IF(wrapper STREQUAL "TIME")
+		SET(WRAPPER_COMMAND ${TIME_TOOL_PATH})
+	ELSEIF(wrapper STREQUAL "MEMCHECK" AND VALGRIND_TOOL_PATH)
+		SET(WRAPPER_COMMAND "${VALGRIND_TOOL_PATH} --tool=memcheck --log-file=${case_path}/${case_name}_memcheck.log -v --leak-check=full --show-reachable=yes --track-origins=yes --malloc-fill=0xff --free-fill=0xff")
+		SET(tester MEMCHECK)
+	ELSEIF(wrapper STREQUAL "CALLGRIND" AND VALGRIND_TOOL_PATH)
+		SET(WRAPPER_COMMAND "${VALGRIND_TOOL_PATH} --tool=callgrind --branch-sim=yes --cache-sim=yes --dump-instr=yes --collect-jumps=yes")
+		UNSET(tester)
+	ENDIF()
+
+	# Implement testers
+	IF(tester STREQUAL "DIFF")
+		SET(TESTER_COMMAND "${DIFF_TOOL_PATH} -sbB ${case_path}/${case_name}_expected_result.vtu ${case_path}/${case_name}_with_results.vtu")
+	ELSEIF(tester STREQUAL "MEMCHECK")
+		SET(TESTER_COMMAND "! ${GREP_TOOL_PATH} definitely ${case_path}/${case_name}_memcheck.log")
+	ENDIF()
+
+	## -----------
+
+	ADD_TEST(NAME "${executable}-${case_path}-${wrapper}"
+		COMMAND ${CMAKE_COMMAND}
+		-Dexecutable=$<TARGET_FILE:${executable}>
+		-Dcase_path=${case_path}
+		-Dcase_name=${case_name}
+		-Dwrapper=${wrapper}
+		-DWRAPPER_COMMAND=${WRAPPER_COMMAND}
+		-DCMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}
+		-P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
+	)
+
+	IF(NOT tester)
+		RETURN()
+	ENDIF()
+
+	ADD_TEST(NAME "${executable}-${case_path}-${wrapper}-${tester}"
+		COMMAND ${CMAKE_COMMAND}
+		-Dcase_path=${case_path}
+		-Dcase_name=${case_name}
+		-Dtester=${tester}
+		-DTESTER_COMMAND=${TESTER_COMMAND}
+		-DCMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}
+		-P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestTester.cmake
+	)
+
+ENDFUNCTION()

scripts/cmake/test/AddTestTester.cmake

+#message("tester: ${TESTER_COMMAND}")
+#STRING(REPLACE " " ";" TESTER_COMMAND ${TESTER_COMMAND})
+#set(list ${TESTER_COMMAND})
+#message("tester: ${list}")
+
+
+EXECUTE_PROCESS(
+	COMMAND bash -c ${TESTER_COMMAND}
+	WORKING_DIRECTORY ${case_path}
+	RESULT_VARIABLE EXIT_CODE
+)
+
+IF(NOT EXIT_CODE STREQUAL "0")
+	MESSAGE(FATAL_ERROR "Error exit code: ${EXIT_CODE}")
+ENDIF()

scripts/cmake/test/AddTestWrapper.cmake

+SET(ARGS --boundary_condition ${case_path}/${case_name}.cnd -g ${case_path}/${case_name}.gml -m ${case_path}/${case_name}.vtu)
+
+STRING(REPLACE " " ";" WRAPPER_COMMAND ${WRAPPER_COMMAND})
+EXECUTE_PROCESS(
+	COMMAND ${WRAPPER_COMMAND} ${executable} ${ARGS}
+	WORKING_DIRECTORY ${case_path}
+	RESULT_VARIABLE EXIT_CODE
+)
+
+IF(NOT EXIT_CODE STREQUAL "0")
+	MESSAGE(FATAL_ERROR "Test wrapper exited with code: ${EXIT_CODE}")
+ENDIF()

scripts/cmake/test/CTestCustom.cmake.in

+# SET(CTEST_CUSTOM_POST_TEST "cat ${CMAKE_BINARY_DIR}/Testing/Temporary/LastTestTester.log 2>&1")
+SET(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE}) # logog

scripts/cmake/test/README.md

+Required tools:
+
+- `time`
+- `diff`
+- `valgrind`
+- `grep`
+- `bash`
+
+Test data is searched for in `ogs6-sources/../ogs6-data`.
+
+In the build directory run `ctest` with `–-output-on-failure` and `-R` for includes and `-E` for excludes:
+
+```bash
+ctest --output-on-failure -R "MEMCHECK|VALGRIND" -E DIFF
+```
+
+Wrapper and tester are implemented in `AddTest.cmake`.

scripts/cmake/test/Test.cmake

+# Find tools and data
+FIND_PROGRAM(TIME_TOOL_PATH time REQUIRED)
+FIND_PROGRAM(DIFF_TOOL_PATH diff REQUIRED)
+FIND_PROGRAM(GREP_TOOL_PATH grep REQUIRED)
+FIND_PROGRAM(BASH_TOOL_PATH bash REQUIRED)
+FIND_PROGRAM(VALGRIND_TOOL_PATH valgrind REQUIRED)
+FIND_FILE(OGS-DATA_PATH ".ogs6-data.dummy" REQUIRED
+	HINTS ${CMAKE_SOURCE_DIR}/../ogs6-data)
+GET_FILENAME_COMPONENT(OGS-DATA_PATH ${OGS-DATA_PATH} PATH)
+
+ENABLE_TESTING() # Enable CTest
+
+# See http://www.vtk.org/Wiki/CMake/Testing_With_CTest for some customization options
+SET(CTEST_CUSTOM_TESTS_IGNORE test-harness) # ignore logog test
+CONFIGURE_FILE(
+	${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/CTestCustom.cmake.in
+	${CMAKE_BINARY_DIR}/CTestCustom.cmake
+)