Merge branch 'scheduled-pipeline' into 'master'

[ci] Pipeline refactoring

See merge request ogs/ogs!4464

.gitlab-ci.yml

@@ -14,55 +14,35 @@ variables:
   CTEST_EXCLUDE_REGEX: "" # Regex is given via ctest -E [regex]
   CPU_TARGET: ivybridge # envinf1 has oldest cpu
   ARTIFACTS_PAGES_URL: https://$CI_PROJECT_ROOT_NAMESPACE.$CI_PAGES_DOMAIN/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts
+  PIPELINE_NAME: "Default pipeline name"
 
 workflow:
+  name: "$PIPELINE_NAME"
   rules:
     # Disable CI for non-MR user-scoped pipelines
     - if: '$CI_MERGE_REQUEST_IID == null && $CI_PROJECT_PATH != "ogs/ogs"'
       when: never
     - if: "$CI_MERGE_REQUEST_LABELS =~ /.*(workflow::paused|ci skip).*/"
       when: never
+    - if: $CI_MERGE_REQUEST_LABELS =~ /.*ci::web only.*/
+      variables:
+        PIPELINE_NAME: "web only MR pipeline: $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"
+        CTEST_INCLUDE_REGEX: "nb-"
     - if: $CI_MERGE_REQUEST_IID # merge requests
+      variables:
+        PIPELINE_NAME: "MR pipeline: $CI_MERGE_REQUEST_SOURCE_PROJECT_PATH $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"
     - if: $CI_COMMIT_TAG # tags, ogs/ogs repo only
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "ogs/ogs"' # master, ogs/ogs repo only
     - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/ && $CI_PROJECT_PATH == "ogs/ogs"' # release branches, e.g. v6.x.x
 
 include:
   - local: "scripts/ci/extends/*.yml"
-  - local: "/scripts/ci/jobs/meta.yml"
-  - local: "/scripts/ci/jobs/ci_images.yml"
-  # jobs, can be indiviually disabled for testing
-  - local: "/scripts/ci/jobs/pre-commit.yml"
-  - local: "/scripts/ci/jobs/build-linux.yml"
-  - local: "/scripts/ci/jobs/build-linux-petsc.yml"
-  - local: "/scripts/ci/jobs/build-linux-frontend.yml"
-  - local: "/scripts/ci/jobs/build-docs.yml"
-  - local: "/scripts/ci/jobs/build-win.yml"
-  - local: "/scripts/ci/jobs/build-mac.yml"
-  - local: "/scripts/ci/jobs/build-container.yml"
-  - local: "/scripts/ci/jobs/jupyter.yml"
-  - local: "/scripts/ci/jobs/code-quality.yml"
-  - local: "/scripts/ci/jobs/code-coverage.yml"
-  - local: "/scripts/ci/jobs/build-gui-linux.yml"
-  - local: "/scripts/ci/jobs/build-gui-win.yml"
-  - local: "/scripts/ci/jobs/build-gui-mac.yml"
-  - local: "/scripts/ci/jobs/check-header.yml"
-  - local: "/scripts/ci/jobs/include-what-you-use.yml"
-  - local: "/scripts/ci/jobs/clang-sanitizer.yml"
-  - local: "/scripts/ci/jobs/clang-tidy.yml"
-  - local: "/scripts/ci/jobs/web.yml"
-  - local: "/scripts/ci/jobs/container.yml"
-  - local: "/scripts/ci/jobs/release.yml"
-  - local: "/scripts/ci/jobs/package.yml"
-  - local: "/scripts/ci/jobs/trigger.yml"
-
-# child pipelines
-wheels:
-  stage: build
-  needs: []
-  trigger:
-    include: /scripts/ci/jobs/build-wheels.yml
-  rules:
-    - if: $CI_COMMIT_TAG
-    - when: manual
-      allow_failure: true
+  - local: "/scripts/ci/pipelines/web.yml"
+    rules:
+      - if: $CI_MERGE_REQUEST_LABELS =~ /.*ci::web only.*/
+  - local: "/scripts/ci/pipelines/regular.yml"
+    rules:
+      - if: $CI_PIPELINE_SOURCE != "schedule" && $CI_MERGE_REQUEST_LABELS !~ /.*ci::web only.*/
+  - local: "/scripts/ci/pipelines/scheduled.yml"
+    rules:
+      - if: '$CI_PIPELINE_SOURCE == "schedule"'

scripts/ci/jobs/build-linux-arch.yml

+build linux arch:
+  extends:
+    - .template-build-linux
+    - .test-artifacts
+  tags: [shell, envinf23]
+  needs: [meta]
+  timeout: 2h
+  variables:
+    BUILD_CTEST_LARGE_ON_MASTER: "true"
+    CMAKE_PRESET: release-all
+    CMAKE_ARGS: >-
+      -DBUILD_SHARED_LIBS=ON
+      -DOGS_USE_MKL=ON

scripts/ci/jobs/build-linux.yml

@@ -62,20 +62,6 @@ build linux (no deps, no procs):
     CMAKE_ARGS: -DOGS_BUILD_PROCESSES=SteadyStateDiffusion
     CMAKE_PRESET: ci-simplest
 
-build linux arch:
-  extends:
-    - .template-build-linux
-    - .test-artifacts
-  tags: [shell, envinf23]
-  needs: [meta]
-  timeout: 2h
-  variables:
-    BUILD_CTEST_LARGE_ON_MASTER: "true"
-    CMAKE_PRESET: release-all
-    CMAKE_ARGS: >-
-      -DBUILD_SHARED_LIBS=ON
-      -DOGS_USE_MKL=ON
-
 build linux debug with sanitizers:
   extends:
     - .template-build-linux

scripts/ci/jobs/trigger.yml

-ufz-bgr/hpc/tutorial:
-  stage: check
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "master"'
-  trigger: ufz-bgr/hpc/tutorial

scripts/ci/jobs/web.yml → scripts/ci/jobs/web-check.yml

@@ -28,65 +28,3 @@ check web links:
       --ignore-url='www.smartkd-concept.de'
       http://localhost:1313
     - kill $!
-
-preview web site:
-  stage: build
-  image: $WEB_IMAGE
-  needs: [ci_images, "build linux arch", "build linux petsc"]
-  variables:
-    HUGO_ENVIRONMENT: "staging"
-  script:
-    # Copy notebook pages
-    - cp -rl build/*/web/content web/
-    - cd web
-    # Symlink for prj link checks (data-link shortcode)
-    - ln -s ../Tests .
-    - yarn
-    - yarn build
-  rules:
-    - if: $CI_MERGE_REQUEST_ID
-      changes:
-        - web/**/*
-        - scripts/ci/jobs/web.yml
-        - Tests/Data/**/*.ipynb
-  artifacts:
-    paths:
-      - web/public
-    expire_in: 1 week
-  environment:
-    name: web preview $CI_MERGE_REQUEST_IID
-    url: ${ARTIFACTS_PAGES_URL}/web/public/index.html
-  cache:
-    paths:
-      - web/node_modules
-
-deploy web site:
-  stage: package
-  image: $WEB_IMAGE
-  needs:
-    - job: ci_images
-    - job: "build linux arch"
-    - job: "build linux petsc"
-    - job: release
-      optional: true
-  rules:
-    - if: $CI_COMMIT_TAG
-    - if: '$CI_COMMIT_BRANCH == "master"'
-      changes:
-        - web/**/*
-        - scripts/ci/jobs/web.yml
-        - Tests/Data/**/*.ipynb
-    - if: '$CI_COMMIT_BRANCH == "master"'
-      when: manual
-      allow_failure: true
-  script:
-    # Copy notebook pages
-    - cp -rl build/*/web/content web/
-    - cd web
-    - yarn
-    - yarn build
-    - netlify deploy --prod --dir=public --site=$OGS_NETLIFY_SITE_ID
-    - yarn upload-index
-  cache:
-    paths:
-      - web/node_modules

scripts/ci/jobs/web-deploy.yml

+deploy web site:
+  stage: package
+  image: $WEB_IMAGE
+  needs:
+    - job: ci_images
+    - job: "build linux arch"
+    - job: "build linux petsc"
+    - job: release
+      optional: true
+  rules:
+    - if: $CI_COMMIT_TAG
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      changes:
+        - web/**/*
+        - scripts/ci/jobs/web.yml
+        - Tests/Data/**/*.ipynb
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: manual
+      allow_failure: true
+  script:
+    # Copy notebook pages
+    - cp -rl build/*/web/content web/
+    - cd web
+    - yarn
+    - yarn build
+    - netlify deploy --prod --dir=public --site=$OGS_NETLIFY_SITE_ID
+    - yarn upload-index
+  cache:
+    paths:
+      - web/node_modules

scripts/ci/jobs/web-preview.yml

+preview web site:
+  stage: build
+  image: $WEB_IMAGE
+  needs: [ci_images, "build linux arch", "build linux petsc"]
+  variables:
+    HUGO_ENVIRONMENT: "staging"
+  script:
+    # Copy notebook pages
+    - cp -rl build/*/web/content web/
+    - cd web
+    # Symlink for prj link checks (data-link shortcode)
+    - ln -s ../Tests .
+    - yarn
+    - yarn build
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+      changes:
+        - web/**/*
+        - scripts/ci/jobs/web.yml
+        - Tests/Data/**/*.ipynb
+  artifacts:
+    paths:
+      - web/public
+    expire_in: 1 week
+  environment:
+    name: web preview $CI_MERGE_REQUEST_IID
+    url: ${ARTIFACTS_PAGES_URL}/web/public/index.html
+  cache:
+    paths:
+      - web/node_modules

scripts/ci/pipelines/regular.yml

+include:
+  - local: "/scripts/ci/jobs/meta.yml"
+  - local: "/scripts/ci/jobs/ci_images.yml"
+  # jobs, can be indiviually disabled for testing
+  - local: "/scripts/ci/jobs/pre-commit.yml"
+  - local: "/scripts/ci/jobs/build-linux.yml"
+  - local: "/scripts/ci/jobs/build-linux-arch.yml"
+  - local: "/scripts/ci/jobs/build-linux-petsc.yml"
+  - local: "/scripts/ci/jobs/build-linux-frontend.yml"
+  - local: "/scripts/ci/jobs/build-docs.yml"
+  - local: "/scripts/ci/jobs/build-win.yml"
+  - local: "/scripts/ci/jobs/build-mac.yml"
+  - local: "/scripts/ci/jobs/build-container.yml"
+  - local: "/scripts/ci/jobs/jupyter.yml"
+  - local: "/scripts/ci/jobs/code-quality.yml"
+  - local: "/scripts/ci/jobs/code-coverage.yml"
+  - local: "/scripts/ci/jobs/build-gui-linux.yml"
+  - local: "/scripts/ci/jobs/build-gui-win.yml"
+  - local: "/scripts/ci/jobs/build-gui-mac.yml"
+  - local: "/scripts/ci/jobs/check-header.yml"
+  - local: "/scripts/ci/jobs/include-what-you-use.yml"
+  - local: "/scripts/ci/jobs/clang-sanitizer.yml"
+  - local: "/scripts/ci/jobs/clang-tidy.yml"
+  - local: "/scripts/ci/jobs/web-preview.yml"
+  - local: "/scripts/ci/jobs/web-check.yml"
+  - local: "/scripts/ci/jobs/web-deploy.yml"
+  - local: "/scripts/ci/jobs/container.yml"
+  - local: "/scripts/ci/jobs/release.yml"
+  - local: "/scripts/ci/jobs/package.yml"
+
+# child pipelines
+wheels:
+  stage: build
+  needs: []
+  trigger:
+    include: /scripts/ci/jobs/build-wheels.yml
+  rules:
+    - if: $CI_COMMIT_TAG
+    - when: manual
+      allow_failure: true
+
+ufz-bgr/hpc/tutorial:
+  stage: check
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+  trigger: ufz-bgr/hpc/tutorial

scripts/ci/pipelines/scheduled.yml

+wheels:
+  stage: build
+  trigger:
+    include: /scripts/ci/jobs/build-wheels.yml

scripts/ci/pipelines/web.yml

+include:
+  - local: "/scripts/ci/extends/*.yml"
+  - local: "/scripts/ci/jobs/meta.yml"
+  - local: "/scripts/ci/jobs/ci_images.yml"
+  - local: "/scripts/ci/jobs/build-linux-arch.yml"
+  - local: "/scripts/ci/jobs/build-linux-petsc.yml"
+  - local: "/scripts/ci/jobs/web-preview.yml"

web/content/docs/devguide/testing/gitlab-ci/index.md

@@ -44,19 +44,30 @@ Or add `[ci skip]` to the commit message to skip the pipeline for this commit. E
 git commit -m "Added feature X [ci skip]"
 ```
 
+Or use the merge request label `ci::skip`.
+
 ## (Temporarily) reduce pipeline run time
 
-During work on MR you may want to reduce the pipeline run time to get feedback faster. You can change the pipeline by editing the `.gitlab-ci.yml` and `scripts/ci/jobs/*.yml` files. It is good practice to mark these changes with commits starting with `drop: ...` in the commit message so they can be removed later easily.
+During work on MR you may want to reduce the pipeline run time to get feedback faster. You can use a pre-configured reduced pipeline or manually editing the pipeline configuration files.
+
+## Pre-configured reduced pipelines
+
+These pipelines can be selected by using merge request labels:
+
+- `ci::web only`: Runs Jupyter notebook ctests only and builds the web site.
+- More to come in the future...
+
+### Manually editing `.gitlab-ci.yml` and `scripts/ci/pipelines/regular.yml`
 
-### `.gitlab-ci.yml`
+You can change the pipeline by editing the `.gitlab-ci.yml` and `scripts/ci/jobs/*.yml` files. It is good practice to mark these changes with commits starting with `drop: ...` in the commit message so they can be removed later easily.
 
-These variables modify the pipeline:
+These variables in `.gitlab-ci.yml` modify the pipeline:
 
 - `BUILD_TESTS`: Set this to `false` to disable unit tests.
 - `BUILD_CTEST`: Set this to `false` to disable ctest (benchmark) tests.
 - `CTEST_INCLUDE_REGEX`: Set this to a regex to select benchmarks to run, e.g. `nb` would select all notebook-based tests and would disable all other benchmarks.
 
-All jobs get included in the `include:`-section. You can simple comment out files to disable jobs defined in that files. Please note that some jobs depend on other jobs.
+All jobs get included in the `include:`-section in the file `scripts/ci/pipelines/regular.yml`. You can simple comment out files to disable jobs defined in that files. Please note that some jobs depend on other jobs.
 
 ### `scripts/ci/jobs/*.yml`