Skip to content
Snippets Groups Projects
Unverified Commit 0c7dd480 authored by Lars Bilke's avatar Lars Bilke
Browse files

[cmake,py] Removed snakemake setup.

parent 89619bad
No related branches found
No related tags found
No related merge requests found
...@@ -26,12 +26,6 @@ find_program(CCACHE_TOOL_PATH ccache) ...@@ -26,12 +26,6 @@ find_program(CCACHE_TOOL_PATH ccache)
find_program(MODULE_CMD lmod PATHS /software/lmod/lmod/libexec) find_program(MODULE_CMD lmod PATHS /software/lmod/lmod/libexec)
find_program(SNAKEMAKE snakemake)
find_program(TEE_TOOL_PATH tee)
if(OGS_BUILD_TESTING AND SNAKEMAKE AND NOT TEE_TOOL_PATH)
message(WARNING "tee tool was not found. Snakemake tests are disabled!")
endif()
find_program(GMSH gmsh) find_program(GMSH gmsh)
find_program(XMLSTARLET_TOOL_PATH xmlstarlet) find_program(XMLSTARLET_TOOL_PATH xmlstarlet)
......
...@@ -92,15 +92,8 @@ if(OGS_USE_PIP) ...@@ -92,15 +92,8 @@ if(OGS_USE_PIP)
file(STRINGS Tests/Data/requirements-dev.txt _requirements_dev) file(STRINGS Tests/Data/requirements-dev.txt _requirements_dev)
list(APPEND OGS_PYTHON_PACKAGES ${_requirements} ${_requirements_dev}) list(APPEND OGS_PYTHON_PACKAGES ${_requirements} ${_requirements_dev})
list( list(APPEND OGS_PYTHON_PACKAGES
APPEND "setuptools" # https://github.com/glenfant/stopit/issues/32
OGS_PYTHON_PACKAGES
"snakemake==${ogs.minimum_version.snakemake}"
"pulp==2.7.0" # https://github.com/snakemake/snakemake/issues/2607
"setuptools" # https://github.com/glenfant/stopit/issues/32
)
set(SNAKEMAKE ${LOCAL_VIRTUALENV_BIN_DIR}/snakemake CACHE FILEPATH ""
FORCE
) )
endif() endif()
endif() endif()
...@@ -209,8 +202,7 @@ function(setup_venv_dependent_ctests) ...@@ -209,8 +202,7 @@ function(setup_venv_dependent_ctests)
PATH Parabolic/HT/InvalidProjectFiles PATH Parabolic/HT/InvalidProjectFiles
EXECUTABLE ogs EXECUTABLE ogs
EXECUTABLE_ARGS ${ht_invalid_prj_file} EXECUTABLE_ARGS ${ht_invalid_prj_file}
RUNTIME 1 RUNTIME 1 PROPERTIES WILL_FAIL TRUE
PROPERTIES WILL_FAIL TRUE
) )
endforeach() endforeach()
endif() endif()
......
...@@ -84,18 +84,14 @@ set_target_properties( ...@@ -84,18 +84,14 @@ set_target_properties(
Testing Testing
) )
configure_file(
${PROJECT_SOURCE_DIR}/scripts/test/buildinfo.in.yaml
${PROJECT_BINARY_DIR}/buildinfo.yaml
)
file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/logs) file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/logs)
# Files in Testing/Notes are submitted to cdash, supply additional build info. # Files in Testing/Notes are submitted to cdash, supply additional build info.
if(DEFINED ENV{CI_JOB_URL}) if(DEFINED ENV{CI_JOB_URL})
file(GENERATE OUTPUT ${PROJECT_BINARY_DIR}/Testing/Notes/buildinfo.txt file(
CONTENT GENERATE
"CI_JOB_URL=$ENV{CI_JOB_URL} OUTPUT ${PROJECT_BINARY_DIR}/Testing/Notes/buildinfo.txt
CONTENT "CI_JOB_URL=$ENV{CI_JOB_URL}
COMMIT_URL=$ENV{CI_PROJECT_URL}/-/commit/$ENV{CI_COMMIT_SHA} COMMIT_URL=$ENV{CI_PROJECT_URL}/-/commit/$ENV{CI_COMMIT_SHA}
CI_COMMIT_TIMESTAMP=$ENV{CI_COMMIT_TIMESTAMP}\n" CI_COMMIT_TIMESTAMP=$ENV{CI_COMMIT_TIMESTAMP}\n"
) )
......
rule generate_square_mesh:
output:
"{mesh_name_prefix,\\w+}_{size,\\d+}_{lx,\\d+}x{ly,\\d+}_{type}.vtu"
shell:
"""
generateStructuredMesh -e {wildcards.type} \
--lx {wildcards.lx} --ly {wildcards.ly} \
--nx {wildcards.size} --ny {wildcards.size} \
-o {output}
"""
dependencies:
- python
"""Snakemake wrapper for vtkdiff."""
__author__ = "Lars Bilke"
__copyright__ = "Copyright 2020, OpenGeoSys Community"
__license__ = "BSD"
from pathlib import Path
from snakemake.shell import shell
# ruff: noqa: F821
output = Path(snakemake.output[0])
if output.exists():
output.unlink()
if snakemake.params.check_mesh:
shell("vtkdiff {snakemake.input.a} {snakemake.input.b} -m > {snakemake.output[0]}")
for field in snakemake.params.fields:
field_a = field[0]
offset = 0
if len(field) == 4:
offset = 1
field_b = field[0 + offset]
abs_tol = field[1 + offset]
rel_tol = field[2 + offset]
shell(
"""
vtkdiff {snakemake.input.a} {snakemake.input.b} \
-a {field_a} -b {field_b} \
--abs {abs_tol} --rel {rel_tol} 2>&1 | tee -a {snakemake.output[0]}
"""
)
from snakemake.utils import min_version
min_version("7.3")
module partmesh:
snakefile: gitlab("bilke/snakemake-partmesh", path="workflow/Snakefile", tag="bbcdc9a721fcda7a1e9619895dce79dfc2b2c905", host="gitlab.opengeosys.org")
config: config
use rule * from partmesh
SOURCE_DIR: "@PROJECT_SOURCE_DIR@"
BUILD_DIR: "@PROJECT_BINARY_DIR@"
BIN_DIR: "@PROJECT_BINARY_DIR@/bin"
Data_SOURCE_DIR: "@PROJECT_SOURCE_DIR@/Tests/Data"
Data_BINARY_DIR: "@PROJECT_BINARY_DIR@/Tests/Data"
+++
date = "2021-03-05T10:49"
title = "Workflow testing"
author = "Lars Bilke"
weight = 1024
[menu]
[menu.devguide]
parent = "testing"
+++
## Introduction
We use the workflow manager [Snakemake](https://snakemake.readthedocs.io) to test workflows which consist of the execution of several steps which are based on each other.
## Requirements
- [Snakemake](https://snakemake.readthedocs.io/en/stable/getting_started/installation.html) >= {{< dataFile "versions.minimum_version.snakemake" >}}
- If you use `OGS_USE_PIP=ON` then `snakemake` is installed in your virtual environment in your build-directory automatically. You can then call it via `.venv/bin/snakemake ...` (inside your build directory).
- On Windows only:
- The `tee`-utility in the `PATH` (can be installed from <https://sourceforge.net/projects/unxutils>)
## Examples
- [ExtractBoundary.smk](https://gitlab.opengeosys.org/ogs/ogs/-/blob/master/Applications/Utils/ExtractBoundary.smk)
- [VoxelGridFromLayers.smk](https://gitlab.opengeosys.org/ogs/ogs/-/blob/master/Applications/Utils/VoxelGridFromLayers.smk)
These example workflows [are added to CTest](https://gitlab.opengeosys.org/ogs/ogs/-/blob/540d0b454c9e3805a81f7c4a1b6ee7565be6845c/Applications/Utils/Tests.cmake#L302-315) as well:
```cmake
if(SNAKEMAKE AND NOT OGS_USE_MPI)
add_test(NAME snakemake_ExtractBoundary
COMMAND ${SNAKEMAKE} -j 1
--configfile ${PROJECT_BINARY_DIR}/buildinfo.yaml
-s ${CMAKE_CURRENT_SOURCE_DIR}/ExtractBoundary.smk
)
add_test(NAME snakemake_VoxelGridFromLayers
# ...
)
add_dependencies(ctest ExtractBoundary Layers2Grid AddFaultToVoxelGrid)
endif()
```
## Modularization
We started on implementing modular rule definitions and tool wrapper in [`scripts/snakemake`](https://gitlab.opengeosys.org/ogs/ogs/-/tree/master/scripts/snakemake).
## Links
- [Snakemake Documentation](https://snakemake.readthedocs.io)
- [Short tutorial](https://snakemake.readthedocs.io/en/stable/tutorial/short.html)
- [Software Carpentry Workshop](https://carpentries-incubator.github.io/workflows-snakemake/index.html)
- [HPC Carpentry Workshop with Snakemake](http://www.hpc-carpentry.org/hpc-python/)
...@@ -19,7 +19,6 @@ ...@@ -19,7 +19,6 @@
"petsc": "3.18.6", "petsc": "3.18.6",
"qt": "5.14.2", "qt": "5.14.2",
"python": "3.9", "python": "3.9",
"snakemake": "8.2.4",
"hdf5": "1.10.7", "hdf5": "1.10.7",
"libxml2": "2.9.12", "libxml2": "2.9.12",
"tfel-rliv": "4.0", "tfel-rliv": "4.0",
...@@ -45,7 +44,6 @@ ...@@ -45,7 +44,6 @@
"python": { "python": {
"jupyter_image": { "jupyter_image": {
"conda_packages": [ "conda_packages": [
"snakemake-minimal",
"pyvista==0.34.2", "pyvista==0.34.2",
"ipyvtklink", "ipyvtklink",
"pythreejs" "pythreejs"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment