Moved from CMake external data to 'real' files.

Test output files are written to ${CMAKE_BUILD_DIR}/Tests/Data.

As a WIP data files have to be in ${CMAKE_SOURCE_DIR}../data

CMakeLists.txt

@@ -189,12 +189,6 @@ if( OGS_BUILD_TESTS AND NOT IS_SUBPROJECT )
 			add_subdirectory( SimpleTests/SolverTests )
 		endif()
 	endif()
-
-	# Create a target 'data', which downloads all referenced data sets into the build tree
-	# This has to be defined after all tests are defined
-	ExternalData_Add_Target(data)
-	set_target_properties(data PROPERTIES EXCLUDE_FROM_ALL 1)
-	add_dependencies(ctest data)
 endif() # OGS_BUILD_TESTS
 
 # The configuration must be called from the source dir and not BaseLib/.

SimpleTests/MeshTests/MPI/CMakeLists.txt

@@ -21,16 +21,14 @@ endif()
 
 set(FilePath "DATA{${ExternalData_SOURCE_ROOT}/NodePartitionedMesh/ASCII/,REGEX:.*}")
 set(MPITestParameters -np 3 "${PROJECT_BINARY_DIR}/bin/test_node_partitioned_mesh" "${FilePath}/mesh_3d")
-ExternalData_Add_Test(
-	data
+Add_Test(
 	NAME NodePartitionedMeshTestASCII
 	COMMAND "mpirun" ${MPITestParameters}
 )
 
 set(FilePath "DATA{${ExternalData_SOURCE_ROOT}/NodePartitionedMesh/Binary/,REGEX:.*}")
 set(MPITestParameters -np 3 "${PROJECT_BINARY_DIR}/bin/test_node_partitioned_mesh" "${FilePath}/mesh_3d")
-ExternalData_Add_Test(
-	data
+Add_Test(
 	NAME NodePartitionedMeshTestBinary
 	COMMAND  "mpirun" ${MPITestParameters}
 )
@@ -38,22 +36,19 @@ ExternalData_Add_Test(
 ## FEM#PETSc solver tests
 set(FilePath "DATA{${ExternalData_SOURCE_ROOT}/EllipticPETSc/,REGEX:.*}")
 set(MPITestParameters -np 3 "${PROJECT_BINARY_DIR}/bin/ogs" "${FilePath}/quad_20x10_GroundWaterFlow.prj" --  -gw_ksp_type bcgs -gw_pc_type bjacobi -gw_ksp_atol 1.e-10)
-ExternalData_Add_Test(
-    data
+Add_Test(
     NAME ParallelFEM_GroundWaterFlow2D
     COMMAND  "mpirun" ${MPITestParameters} ${SolverType}
 )
 
 set(MPITestParameters -np 3 "${PROJECT_BINARY_DIR}/bin/ogs" "${FilePath}/cube_1e3.prj" --  -gw_ksp_type bcgs -gw_pc_type bjacobi -gw_ksp_atol 1.e-10)
-ExternalData_Add_Test(
-    data
+Add_Test(
     NAME ParallelFEM_GroundWaterFlow3D_DirichletBC
     COMMAND  "mpirun" ${MPITestParameters} ${SolverType}
 )
 
 set(MPITestParameters -np 3 "${PROJECT_BINARY_DIR}/bin/ogs" "${FilePath}/cube_1e3_neumann.prj" --  -gw_ksp_type bcgs -gw_pc_type bjacobi -gw_ksp_atol 1.e-10)
-ExternalData_Add_Test(
-    data
+Add_Test(
     NAME ParallelFEM_GroundWaterFlow3D_NeumannBC
     COMMAND  "mpirun" ${MPITestParameters} ${SolverType}
 )

scripts/cmake/test/AddTest.cmake

@@ -31,13 +31,22 @@
 
 function (AddTest)
 
+	set(ExternalData_SOURCE_ROOT ${CMAKE_SOURCE_DIR}/../data)
+	set(ExternalData_BINARY_ROOT ${CMAKE_BINARY_DIR}/Tests/Data)
+
 	# parse arguments
 	set(options NONE)
 	set(oneValueArgs EXECUTABLE PATH NAME WRAPPER TESTER)
 	set(multiValueArgs EXECUTABLE_ARGS DATA DIFF_DATA WRAPPER_ARGS)
 	cmake_parse_arguments(AddTest "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-	set(AddTest_SOURCE_PATH "${ExternalData_SOURCE_ROOT}/${AddTest_PATH}")
-	set(AddTest_BINARY_PATH "${ExternalData_BINARY_ROOT}/${AddTest_PATH}")
+
+	# TODO set to submodule
+	set(AddTest_SOURCE_PATH "${CMAKE_SOURCE_DIR}/../data/${AddTest_PATH}")
+	set(AddTest_BINARY_PATH "${CMAKE_BINARY_DIR}/Tests/Data/${AddTest_PATH}")
+	file(MAKE_DIRECTORY ${AddTest_BINARY_PATH})
+	file(TO_NATIVE_PATH "${AddTest_BINARY_PATH}" AddTest_BINARY_PATH_NATIVE)
+
+	set(AddTest_EXECUTABLE_ARGS "${AddTest_EXECUTABLE_ARGS} -o ${AddTest_BINARY_PATH_NATIVE}")
 
 	# set defaults
 	if(NOT AddTest_EXECUTABLE)
@@ -48,20 +57,6 @@ function (AddTest)
 		set(AddTest_WRAPPER time)
 	endif()
 
-	# replace arguments which reference test data files with the correct DATA{}-path
-	foreach(ARG ${AddTest_EXECUTABLE_ARGS})
-		string(REGEX MATCH ".*${ARG}.*" ARG_FOUND ${AddTest_DATA} )
-		if(ARG_FOUND)
-			set(AddTest_EXECUTABLE_ARGS_PARSED ${AddTest_EXECUTABLE_ARGS_PARSED} DATA{${AddTest_SOURCE_PATH}/${ARG}})
-		else()
-			set(AddTest_EXECUTABLE_ARGS_PARSED ${AddTest_EXECUTABLE_ARGS_PARSED} ${ARG}})
-		endif()
-	endforeach()
-
-	string(REPLACE ";" "," AddTest_DATA "${AddTest_DATA}")
-	set(AddTest_DATA "${AddTest_SOURCE_PATH}/${AddTest_DATA}")
-
-
 	# --- Implement wrappers ---
 	# check requirements, disable if not met
 	if(AddTest_WRAPPER STREQUAL "time" AND NOT TIME_TOOL_PATH)
@@ -124,7 +119,9 @@ function (AddTest)
 			get_filename_component(FILE_NAME ${FILE} NAME_WE)
 			get_filename_component(FILE_EXT ${FILE} EXT)
 			set(FILE_EXPECTED ${FILE_NAME}_expected${FILE_EXT})
-			set(TESTER_COMMAND ${TESTER_COMMAND} "${SELECTED_DIFF_TOOL_PATH} ${TESTER_ARGS} DATA{${AddTest_SOURCE_PATH}/${FILE_EXPECTED}} ${AddTest_BINARY_PATH}/${FILE}")
+			set(TESTER_COMMAND ${TESTER_COMMAND} "${SELECTED_DIFF_TOOL_PATH} \
+				${TESTER_ARGS} ${AddTest_SOURCE_PATH}/${FILE_EXPECTED} \
+				${AddTest_BINARY_PATH}/${FILE}")
 			if(AddTest_DIFF_DATA_PARSED)
 				set(AddTest_DIFF_DATA_PARSED "${AddTest_DIFF_DATA_PARSED},${FILE_EXPECTED}")
 			else()
@@ -143,7 +140,8 @@ function (AddTest)
 		list(GET AddTest_DIFF_DATA 2 NAME_B)
 
 		set(TESTER_COMMAND ${TESTER_COMMAND} "${SELECTED_DIFF_TOOL_PATH} \
-			${AddTest_BINARY_PATH}/${VTK_FILE} -a ${NAME_A} -b ${NAME_B} \
+			${AddTest_BINARY_PATH}/${VTK_FILE} \
+			-a ${NAME_A} -b ${NAME_B} \
 			${TESTER_ARGS}")
 		string(REPLACE ";" " && " TESTER_COMMAND "${TESTER_COMMAND}")
 	elseif(tester STREQUAL "memcheck")
@@ -158,17 +156,14 @@ function (AddTest)
 	endif()
 
 	# Run the wrapper
-	ExternalData_Add_Test(data
+	Add_Test(
 		NAME "${AddTest_EXECUTABLE}-${AddTest_NAME}-${AddTest_WRAPPER}"
 		COMMAND ${CMAKE_COMMAND}
 		-DEXECUTABLE=${AddTest_EXECUTABLE_PARSED}
-		-DEXECUTABLE_ARGS=${AddTest_EXECUTABLE_ARGS_PARSED}
+		-DEXECUTABLE_ARGS=${AddTest_EXECUTABLE_ARGS}
 		-Dcase_path=${AddTest_SOURCE_PATH}
-		-Dcase_name=${AddTest_NAME}
 		-DWRAPPER_COMMAND=${WRAPPER_COMMAND}
-		-DCMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}
 		-P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestWrapper.cmake
-		DATA{${AddTest_DATA}}
 	)
 
 	if(NOT AddTest_TESTER)
@@ -177,13 +172,11 @@ function (AddTest)
 
 	# Run the tester
 	if(AddTest_TESTER STREQUAL "diff" OR AddTest_TESTER STREQUAL "numdiff")
-		ExternalData_Add_Test(data
+		Add_Test(
 			NAME "${AddTest_EXECUTABLE}-${AddTest_NAME}-${AddTest_WRAPPER}-${AddTest_TESTER}"
 			COMMAND ${CMAKE_COMMAND}
 			-Dcase_path=${AddTest_SOURCE_PATH}
-			-Dcase_name=${AddTest_NAME}
 			-DTESTER_COMMAND=${TESTER_COMMAND}
-			-DCMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}
 			-P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestTester.cmake
 			DATA{${AddTest_DIFF_DATA_PARSED}}
 		)
@@ -192,9 +185,7 @@ function (AddTest)
 			NAME "${AddTest_EXECUTABLE}-${AddTest_NAME}-${AddTest_WRAPPER}-${AddTest_TESTER}"
 			COMMAND ${CMAKE_COMMAND}
 			-Dcase_path=${AddTest_SOURCE_PATH}
-			-Dcase_name=${AddTest_NAME}
 			-DTESTER_COMMAND=${TESTER_COMMAND}
-			-DCMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}
 			-P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestTester.cmake
 		)
 	else()
@@ -202,9 +193,7 @@ function (AddTest)
 			NAME "${AddTest_EXECUTABLE}-${AddTest_NAME}-${AddTest_WRAPPER}-${AddTest_TESTER}"
 			COMMAND ${CMAKE_COMMAND}
 			-Dcase_path=${AddTest_SOURCE_PATH}
-			-Dcase_name=${AddTest_NAME}
 			-DTESTER_COMMAND=${TESTER_COMMAND}
-			-DCMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}
 			-P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/AddTestTester.cmake
 		)
 	endif()

scripts/cmake/test/AddTestTester.cmake

-#message("tester: ${TESTER_COMMAND}")
-#string(REPLACE " " ";" TESTER_COMMAND ${TESTER_COMMAND})
-#set(list ${TESTER_COMMAND})
-#message("tester: ${list}")
-
-
 execute_process(
 	COMMAND bash -c ${TESTER_COMMAND}
 	WORKING_DIRECTORY ${case_path}

scripts/cmake/test/AddTestWrapper.cmake

 string(REPLACE " " ";" WRAPPER_COMMAND ${WRAPPER_COMMAND})
+string(REPLACE " " ";" EXECUTABLE_ARGS ${EXECUTABLE_ARGS})
 execute_process(
 	COMMAND ${WRAPPER_COMMAND} ${EXECUTABLE} ${EXECUTABLE_ARGS}
 	WORKING_DIRECTORY ${case_path}

scripts/cmake/test/Data.cmake

-include(ExternalData)
-
-set(ExternalData_LOCAL_FOLDER ogs6-data)
-
-set(ExternalData_OBJECT_STORES "${ExternalData_OBJECT_STORES_DEFAULT}" CACHE STRING
-	"Semicolon-separated list of local directories holding test data files in the layout %(algo)/%(hash).")
-mark_as_advanced(ExternalData_OBJECT_STORES)
-if(NOT ExternalData_OBJECT_STORES)
-	set(ExternalData_OBJECT_STORES "${CMAKE_SOURCE_DIR}/../${ExternalData_LOCAL_FOLDER}")
-	file(MAKE_DIRECTORY "${ExternalData_OBJECT_STORES}")
-endif()
-
-set(ExternalData_SOURCE_ROOT ${CMAKE_SOURCE_DIR}/Tests/Data)
-set(ExternalData_BINARY_ROOT ${CMAKE_BINARY_DIR}/Tests/Data)
-set(ExternalData_LINK_CONTENT MD5)
-
-# Amazon S3 config
-set(ExternalData_S3_Bucket opengeosys)
-
-set(ExternalData_URL_TEMPLATES
-	"http://www.opengeosys.org/images/dev/%(algo)/%(hash)"
-	"http://${ExternalData_S3_Bucket}.s3.amazonaws.com/${ExternalData_LOCAL_FOLDER}/%(algo)/%(hash)"
-)
-
-add_custom_target(
-	move-data
-	COMMAND ${CMAKE_COMMAND}
-	-DExternalData_SOURCE_ROOT=${ExternalData_SOURCE_ROOT}
-	-DExternalData_LINK_CONTENT=${ExternalData_LINK_CONTENT}
-	-DExternalData_OBJECT_STORES=${ExternalData_OBJECT_STORES}
-	-P ${PROJECT_SOURCE_DIR}/scripts/cmake/test/MoveDataToStore.cmake
-	VERBATIM
-)
-
-if(S3CMD_TOOL_PATH)
-	add_custom_target(
-		upload-data
-		COMMAND ${S3CMD_TOOL_PATH} sync --acl-public --skip-existing --no-check-md5
-			${CMAKE_SOURCE_DIR}/../${ExternalData_LOCAL_FOLDER} s3://${ExternalData_S3_Bucket}
-	)
-endif()
-
-if(HOSTNAME STREQUAL "envinf1.eve.ufz.de")
-	add_custom_target(
-		sync-data
-		COMMAND ${CMAKE_COMMAND} -E copy_directory
-		${CMAKE_SOURCE_DIR}/../${ExternalData_LOCAL_FOLDER}
-		/data/ogs/${ExternalData_LOCAL_FOLDER}
-	)
-	if(CURL_TOOL_PATH)
-		add_custom_command(
-			TARGET sync-data POST_BUILD
-			COMMAND ${CURL_TOOL_PATH} --insecure 'https://svn.ufz.de:8443/buildByToken/build?job=Tmp_Trigger&token=ogsbuild&cause=Triggered_by_sync-data_target_on_envinf1'
-			COMMENT "Triggered sync to opengeosys.org, see https://svn.ufz.de:8443/job/OGS-6/job/SyncExternalData"
-		)
-	else()
-		message(STATUS "curl tool was not found but is required for the sync-data target!")
-	endif()
-endif()

scripts/cmake/test/Test.cmake

@@ -40,7 +40,6 @@ configure_file(
 )
 
 include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/AddTest.cmake)
-include(${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/test/Data.cmake)
 
 if(CMAKE_CONFIGURATION_TYPES)
 	set(CONFIG_PARAMETER --build-config "$<CONFIGURATION>")
@@ -53,7 +52,7 @@ add_custom_target(
 	--output-on-failure --output-log Tests/ctest.log
 	--exclude-regex LARGE
 	${CONFIG_PARAMETER} --parallel ${NUM_PROCESSORS} --test-action test
-	DEPENDS data ogs vtkdiff ctest-cleanup
+	DEPENDS ogs vtkdiff ctest-cleanup
 )
 add_custom_target(ctest-large-cleanup ${CMAKE_COMMAND} -E remove Tests/ctest-large.log)
 add_custom_target(