diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7079f9995e994d400f9af201a11c5b28c7c55327..1af2f5e87d886c19335abc88e562ca3f616febc1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,55 +14,35 @@ variables:
CTEST_EXCLUDE_REGEX: "" # Regex is given via ctest -E [regex]
CPU_TARGET: ivybridge # envinf1 has oldest cpu
ARTIFACTS_PAGES_URL: https://$CI_PROJECT_ROOT_NAMESPACE.$CI_PAGES_DOMAIN/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts
+ PIPELINE_NAME: "Default pipeline name"
workflow:
+ name: "$PIPELINE_NAME"
rules:
# Disable CI for non-MR user-scoped pipelines
- if: '$CI_MERGE_REQUEST_IID == null && $CI_PROJECT_PATH != "ogs/ogs"'
when: never
- if: "$CI_MERGE_REQUEST_LABELS =~ /.*(workflow::paused|ci skip).*/"
when: never
+ - if: $CI_MERGE_REQUEST_LABELS =~ /.*ci::web only.*/
+ variables:
+ PIPELINE_NAME: "web only MR pipeline: $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"
+ CTEST_INCLUDE_REGEX: "nb-"
- if: $CI_MERGE_REQUEST_IID # merge requests
+ variables:
+ PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"
- if: $CI_COMMIT_TAG # tags, ogs/ogs repo only
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "ogs/ogs"' # master, ogs/ogs repo only
- if: '$CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/ && $CI_PROJECT_PATH == "ogs/ogs"' # release branches, e.g. v6.x.x
include:
- local: "scripts/ci/extends/*.yml"
- - local: "/scripts/ci/jobs/meta.yml"
- - local: "/scripts/ci/jobs/ci_images.yml"
- # jobs, can be indiviually disabled for testing
- - local: "/scripts/ci/jobs/pre-commit.yml"
- - local: "/scripts/ci/jobs/build-linux.yml"
- - local: "/scripts/ci/jobs/build-linux-petsc.yml"
- - local: "/scripts/ci/jobs/build-linux-frontend.yml"
- - local: "/scripts/ci/jobs/build-docs.yml"
- - local: "/scripts/ci/jobs/build-win.yml"
- - local: "/scripts/ci/jobs/build-mac.yml"
- - local: "/scripts/ci/jobs/build-container.yml"
- - local: "/scripts/ci/jobs/jupyter.yml"
- - local: "/scripts/ci/jobs/code-quality.yml"
- - local: "/scripts/ci/jobs/code-coverage.yml"
- - local: "/scripts/ci/jobs/build-gui-linux.yml"
- - local: "/scripts/ci/jobs/build-gui-win.yml"
- - local: "/scripts/ci/jobs/build-gui-mac.yml"
- - local: "/scripts/ci/jobs/check-header.yml"
- - local: "/scripts/ci/jobs/include-what-you-use.yml"
- - local: "/scripts/ci/jobs/clang-sanitizer.yml"
- - local: "/scripts/ci/jobs/clang-tidy.yml"
- - local: "/scripts/ci/jobs/web.yml"
- - local: "/scripts/ci/jobs/container.yml"
- - local: "/scripts/ci/jobs/release.yml"
- - local: "/scripts/ci/jobs/package.yml"
- - local: "/scripts/ci/jobs/trigger.yml"
-
-# child pipelines
-wheels:
- stage: build
- needs: []
- trigger:
- include: /scripts/ci/jobs/build-wheels.yml
- rules:
- - if: $CI_COMMIT_TAG
- - when: manual
- allow_failure: true
+ - local: "/scripts/ci/pipelines/web.yml"
+ rules:
+ - if: $CI_MERGE_REQUEST_LABELS =~ /.*ci::web only.*/
+ - local: "/scripts/ci/pipelines/regular.yml"
+ rules:
+ - if: $CI_PIPELINE_SOURCE != "schedule" && $CI_MERGE_REQUEST_LABELS !~ /.*ci::web only.*/
+ - local: "/scripts/ci/pipelines/scheduled.yml"
+ rules:
+ - if: '$CI_PIPELINE_SOURCE == "schedule"'
diff --git a/scripts/ci/jobs/build-linux-arch.yml b/scripts/ci/jobs/build-linux-arch.yml
new file mode 100644
index 0000000000000000000000000000000000000000..4245d7b584ff6d11cbf37cf1a58bcd60a42902f2
--- /dev/null
+++ b/scripts/ci/jobs/build-linux-arch.yml
@@ -0,0 +1,13 @@
+build linux arch:
+ extends:
+ - .template-build-linux
+ - .test-artifacts
+ tags: [shell, envinf23]
+ needs: [meta]
+ timeout: 2h
+ variables:
+ BUILD_CTEST_LARGE_ON_MASTER: "true"
+ CMAKE_PRESET: release-all
+ CMAKE_ARGS: >-
+ -DBUILD_SHARED_LIBS=ON
+ -DOGS_USE_MKL=ON
diff --git a/scripts/ci/jobs/build-linux.yml b/scripts/ci/jobs/build-linux.yml
index b9942ff1356ab686021ebb16bf107dc1e5bef2bb..08ac74aed4f6895f1538b5986495a236a8bb6328 100644
--- a/scripts/ci/jobs/build-linux.yml
+++ b/scripts/ci/jobs/build-linux.yml
@@ -62,20 +62,6 @@ build linux (no deps, no procs):
CMAKE_ARGS: -DOGS_BUILD_PROCESSES=SteadyStateDiffusion
CMAKE_PRESET: ci-simplest
-build linux arch:
- extends:
- - .template-build-linux
- - .test-artifacts
- tags: [shell, envinf23]
- needs: [meta]
- timeout: 2h
- variables:
- BUILD_CTEST_LARGE_ON_MASTER: "true"
- CMAKE_PRESET: release-all
- CMAKE_ARGS: >-
- -DBUILD_SHARED_LIBS=ON
- -DOGS_USE_MKL=ON
-
build linux debug with sanitizers:
extends:
- .template-build-linux
diff --git a/scripts/ci/jobs/trigger.yml b/scripts/ci/jobs/trigger.yml
deleted file mode 100644
index 198e5e38fdb8e4374c018b95cf9cca045b754f6c..0000000000000000000000000000000000000000
--- a/scripts/ci/jobs/trigger.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-ufz-bgr/hpc/tutorial:
- stage: check
- rules:
- - if: '$CI_COMMIT_BRANCH == "master"'
- trigger: ufz-bgr/hpc/tutorial
diff --git a/scripts/ci/jobs/web-check.yml b/scripts/ci/jobs/web-check.yml
new file mode 100644
index 0000000000000000000000000000000000000000..239bbf8846c8ec45974c9aaeca00b76e1ab1d690
--- /dev/null
+++ b/scripts/ci/jobs/web-check.yml
@@ -0,0 +1,30 @@
+check web links:
+ stage: check
+ allow_failure: true
+ tags: [docker]
+ extends:
+ - .rules-master-manual
+ needs: [ci_images, "build linux arch", "build linux petsc"]
+ image: $WEB_IMAGE
+ script:
+ # Copy notebook pages
+ - cp -rl build/*/web/content web/
+ - cd web
+ - hugo server &
+ - sleep 20
+ - >
+ linkchecker --no-warnings
+ --check-extern
+ --ignore-url='sciencedirect.com'
+ --ignore-url='wiley.com'
+ --ignore-url='doi.org'
+ --ignore-url='linkinghub.elsevier.com'
+ --ignore-url='overleaf.com'
+ --ignore-url='https://gitlab.opengeosys.org/ogs/ogs/-/commit/'
+ --ignore-url='https://gitlab.opengeosys.org/ogs/ogs/-/merge_requests/new'
+ --ignore-url='https://ogsstorage.blob.core.windows.net/binaries/ogs6'
+ --ignore-url='/css/all.css'
+ --ignore-url='www.grs.de'
+ --ignore-url='www.smartkd-concept.de'
+ http://localhost:1313
+ - kill $!
diff --git a/scripts/ci/jobs/web-deploy.yml b/scripts/ci/jobs/web-deploy.yml
new file mode 100644
index 0000000000000000000000000000000000000000..661c2dc4fb027fec5d9acde5a0c51e93f91a67dc
--- /dev/null
+++ b/scripts/ci/jobs/web-deploy.yml
@@ -0,0 +1,30 @@
+deploy web site:
+ stage: package
+ image: $WEB_IMAGE
+ needs:
+ - job: ci_images
+ - job: "build linux arch"
+ - job: "build linux petsc"
+ - job: release
+ optional: true
+ rules:
+ - if: $CI_COMMIT_TAG
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ changes:
+ - web/**/*
+ - scripts/ci/jobs/web.yml
+ - Tests/Data/**/*.ipynb
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ when: manual
+ allow_failure: true
+ script:
+ # Copy notebook pages
+ - cp -rl build/*/web/content web/
+ - cd web
+ - yarn
+ - yarn build
+ - netlify deploy --prod --dir=public --site=$OGS_NETLIFY_SITE_ID
+ - yarn upload-index
+ cache:
+ paths:
+ - web/node_modules
diff --git a/scripts/ci/jobs/web-preview.yml b/scripts/ci/jobs/web-preview.yml
new file mode 100644
index 0000000000000000000000000000000000000000..646480f69af1be51135ae15ed555ac306c8a6cf0
--- /dev/null
+++ b/scripts/ci/jobs/web-preview.yml
@@ -0,0 +1,30 @@
+preview web site:
+ stage: build
+ image: $WEB_IMAGE
+ needs: [ci_images, "build linux arch", "build linux petsc"]
+ variables:
+ HUGO_ENVIRONMENT: "staging"
+ script:
+ # Copy notebook pages
+ - cp -rl build/*/web/content web/
+ - cd web
+ # Symlink for prj link checks (data-link shortcode)
+ - ln -s ../Tests .
+ - yarn
+ - yarn build
+ rules:
+ - if: $CI_MERGE_REQUEST_ID
+ changes:
+ - web/**/*
+ - scripts/ci/jobs/web.yml
+ - Tests/Data/**/*.ipynb
+ artifacts:
+ paths:
+ - web/public
+ expire_in: 1 week
+ environment:
+ name: web preview $CI_MERGE_REQUEST_IID
+ url: ${ARTIFACTS_PAGES_URL}/web/public/index.html
+ cache:
+ paths:
+ - web/node_modules
diff --git a/scripts/ci/jobs/web.yml b/scripts/ci/jobs/web.yml
deleted file mode 100644
index 2d089e7d1841fd16eed771d6ae3000e533d49354..0000000000000000000000000000000000000000
--- a/scripts/ci/jobs/web.yml
+++ /dev/null
@@ -1,92 +0,0 @@
-check web links:
- stage: check
- allow_failure: true
- tags: [docker]
- extends:
- - .rules-master-manual
- needs: [ci_images, "build linux arch", "build linux petsc"]
- image: $WEB_IMAGE
- script:
- # Copy notebook pages
- - cp -rl build/*/web/content web/
- - cd web
- - hugo server &
- - sleep 20
- - >
- linkchecker --no-warnings
- --check-extern
- --ignore-url='sciencedirect.com'
- --ignore-url='wiley.com'
- --ignore-url='doi.org'
- --ignore-url='linkinghub.elsevier.com'
- --ignore-url='overleaf.com'
- --ignore-url='https://gitlab.opengeosys.org/ogs/ogs/-/commit/'
- --ignore-url='https://gitlab.opengeosys.org/ogs/ogs/-/merge_requests/new'
- --ignore-url='https://ogsstorage.blob.core.windows.net/binaries/ogs6'
- --ignore-url='/css/all.css'
- --ignore-url='www.grs.de'
- --ignore-url='www.smartkd-concept.de'
- http://localhost:1313
- - kill $!
-
-preview web site:
- stage: build
- image: $WEB_IMAGE
- needs: [ci_images, "build linux arch", "build linux petsc"]
- variables:
- HUGO_ENVIRONMENT: "staging"
- script:
- # Copy notebook pages
- - cp -rl build/*/web/content web/
- - cd web
- # Symlink for prj link checks (data-link shortcode)
- - ln -s ../Tests .
- - yarn
- - yarn build
- rules:
- - if: $CI_MERGE_REQUEST_ID
- changes:
- - web/**/*
- - scripts/ci/jobs/web.yml
- - Tests/Data/**/*.ipynb
- artifacts:
- paths:
- - web/public
- expire_in: 1 week
- environment:
- name: web preview $CI_MERGE_REQUEST_IID
- url: ${ARTIFACTS_PAGES_URL}/web/public/index.html
- cache:
- paths:
- - web/node_modules
-
-deploy web site:
- stage: package
- image: $WEB_IMAGE
- needs:
- - job: ci_images
- - job: "build linux arch"
- - job: "build linux petsc"
- - job: release
- optional: true
- rules:
- - if: $CI_COMMIT_TAG
- - if: '$CI_COMMIT_BRANCH == "master"'
- changes:
- - web/**/*
- - scripts/ci/jobs/web.yml
- - Tests/Data/**/*.ipynb
- - if: '$CI_COMMIT_BRANCH == "master"'
- when: manual
- allow_failure: true
- script:
- # Copy notebook pages
- - cp -rl build/*/web/content web/
- - cd web
- - yarn
- - yarn build
- - netlify deploy --prod --dir=public --site=$OGS_NETLIFY_SITE_ID
- - yarn upload-index
- cache:
- paths:
- - web/node_modules
diff --git a/scripts/ci/pipelines/regular.yml b/scripts/ci/pipelines/regular.yml
new file mode 100644
index 0000000000000000000000000000000000000000..778cfc007da92856df8f5d545ce2691c4a0b7551
--- /dev/null
+++ b/scripts/ci/pipelines/regular.yml
@@ -0,0 +1,46 @@
+include:
+ - local: "/scripts/ci/jobs/meta.yml"
+ - local: "/scripts/ci/jobs/ci_images.yml"
+ # jobs, can be indiviually disabled for testing
+ - local: "/scripts/ci/jobs/pre-commit.yml"
+ - local: "/scripts/ci/jobs/build-linux.yml"
+ - local: "/scripts/ci/jobs/build-linux-arch.yml"
+ - local: "/scripts/ci/jobs/build-linux-petsc.yml"
+ - local: "/scripts/ci/jobs/build-linux-frontend.yml"
+ - local: "/scripts/ci/jobs/build-docs.yml"
+ - local: "/scripts/ci/jobs/build-win.yml"
+ - local: "/scripts/ci/jobs/build-mac.yml"
+ - local: "/scripts/ci/jobs/build-container.yml"
+ - local: "/scripts/ci/jobs/jupyter.yml"
+ - local: "/scripts/ci/jobs/code-quality.yml"
+ - local: "/scripts/ci/jobs/code-coverage.yml"
+ - local: "/scripts/ci/jobs/build-gui-linux.yml"
+ - local: "/scripts/ci/jobs/build-gui-win.yml"
+ - local: "/scripts/ci/jobs/build-gui-mac.yml"
+ - local: "/scripts/ci/jobs/check-header.yml"
+ - local: "/scripts/ci/jobs/include-what-you-use.yml"
+ - local: "/scripts/ci/jobs/clang-sanitizer.yml"
+ - local: "/scripts/ci/jobs/clang-tidy.yml"
+ - local: "/scripts/ci/jobs/web-preview.yml"
+ - local: "/scripts/ci/jobs/web-check.yml"
+ - local: "/scripts/ci/jobs/web-deploy.yml"
+ - local: "/scripts/ci/jobs/container.yml"
+ - local: "/scripts/ci/jobs/release.yml"
+ - local: "/scripts/ci/jobs/package.yml"
+
+# child pipelines
+wheels:
+ stage: build
+ needs: []
+ trigger:
+ include: /scripts/ci/jobs/build-wheels.yml
+ rules:
+ - if: $CI_COMMIT_TAG
+ - when: manual
+ allow_failure: true
+
+ufz-bgr/hpc/tutorial:
+ stage: check
+ rules:
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ trigger: ufz-bgr/hpc/tutorial
diff --git a/scripts/ci/pipelines/scheduled.yml b/scripts/ci/pipelines/scheduled.yml
new file mode 100644
index 0000000000000000000000000000000000000000..90a909b0f747f54becab4fb607e450658a74ff39
--- /dev/null
+++ b/scripts/ci/pipelines/scheduled.yml
@@ -0,0 +1,4 @@
+wheels:
+ stage: build
+ trigger:
+ include: /scripts/ci/jobs/build-wheels.yml
diff --git a/scripts/ci/pipelines/web.yml b/scripts/ci/pipelines/web.yml
new file mode 100644
index 0000000000000000000000000000000000000000..579d9635edb5a4257ee43f654290bb66be7eff8a
--- /dev/null
+++ b/scripts/ci/pipelines/web.yml
@@ -0,0 +1,7 @@
+include:
+ - local: "/scripts/ci/extends/*.yml"
+ - local: "/scripts/ci/jobs/meta.yml"
+ - local: "/scripts/ci/jobs/ci_images.yml"
+ - local: "/scripts/ci/jobs/build-linux-arch.yml"
+ - local: "/scripts/ci/jobs/build-linux-petsc.yml"
+ - local: "/scripts/ci/jobs/web-preview.yml"
diff --git a/web/content/docs/devguide/testing/gitlab-ci/index.md b/web/content/docs/devguide/testing/gitlab-ci/index.md
index 8f6b72ac0ddadcdecfea2dd3fde659afd6a13ee3..81333dbcc174eebb73d4e8f0c3ebc9a1069ab458 100644
--- a/web/content/docs/devguide/testing/gitlab-ci/index.md
+++ b/web/content/docs/devguide/testing/gitlab-ci/index.md
@@ -44,19 +44,30 @@ Or add `[ci skip]` to the commit message to skip the pipeline for this commit. E
git commit -m "Added feature X [ci skip]"
```
+Or use the merge request label `ci::skip`.
+
## (Temporarily) reduce pipeline run time
-During work on MR you may want to reduce the pipeline run time to get feedback faster. You can change the pipeline by editing the `.gitlab-ci.yml` and `scripts/ci/jobs/*.yml` files. It is good practice to mark these changes with commits starting with `drop: ...` in the commit message so they can be removed later easily.
+During work on MR you may want to reduce the pipeline run time to get feedback faster. You can use a pre-configured reduced pipeline or manually editing the pipeline configuration files.
+
+## Pre-configured reduced pipelines
+
+These pipelines can be selected by using merge request labels:
+
+- `ci::web only`: Runs Jupyter notebook ctests only and builds the web site.
+- More to come in the future...
+
+### Manually editing `.gitlab-ci.yml` and `scripts/ci/pipelines/regular.yml`
-### `.gitlab-ci.yml`
+You can change the pipeline by editing the `.gitlab-ci.yml` and `scripts/ci/jobs/*.yml` files. It is good practice to mark these changes with commits starting with `drop: ...` in the commit message so they can be removed later easily.
-These variables modify the pipeline:
+These variables in `.gitlab-ci.yml` modify the pipeline:
- `BUILD_TESTS`: Set this to `false` to disable unit tests.
- `BUILD_CTEST`: Set this to `false` to disable ctest (benchmark) tests.
- `CTEST_INCLUDE_REGEX`: Set this to a regex to select benchmarks to run, e.g. `nb` would select all notebook-based tests and would disable all other benchmarks.
-All jobs get included in the `include:`-section. You can simple comment out files to disable jobs defined in that files. Please note that some jobs depend on other jobs.
+All jobs get included in the `include:`-section in the file `scripts/ci/pipelines/regular.yml`. You can simple comment out files to disable jobs defined in that files. Please note that some jobs depend on other jobs.
### `scripts/ci/jobs/*.yml`