[ci] Streamlined Docker image selection logic.

.gitlab-ci.yml

@@ -6,13 +6,6 @@ stages:
   - package
 
 variables:
-  # Docker image names
-  CONTAINER_GCC_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/gcc
-  CONTAINER_GCC_GUI_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/gcc-gui
-  CONTAINER_CLANG_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/clang
-  XUNIT_TO_JUNIT_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/xunit-to-junit
-  PRECOMMIT_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/pre-commit
-  WEB_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/web
   # Build config
   BUILD_TYPE: Release
   BUILD_PROCESSES: "" # Empty string: builds all processes
@@ -32,18 +25,11 @@ workflow:
     - if: $CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/  # release branches, e.g. v6.x.x
 
 include:
-  # extends
-  - local: '/scripts/ci/extends/defaults.yml'
-  - local: '/scripts/ci/extends/rules.yml'
-  - local: '/scripts/ci/extends/template-build-linux.yml'
-  - local: '/scripts/ci/extends/template-build-win.yml'
-  - local: '/scripts/ci/extends/container-maker-setup.yml'
-  - local: '/scripts/ci/extends/vs2019-environment.yml'
-  - local: '/scripts/ci/extends/test-artifacts.yml'
+  - local: 'scripts/ci/extends/*.yml'
   - local: '/scripts/ci/jobs/meta.yml'
+  - local: '/scripts/ci/jobs/ci_images.yml'
   - local: '/scripts/ci/jobs/pre-commit.yml'
   # jobs, can be indiviually disabled for testing
-  - local: '/scripts/ci/jobs/build_image.yml'
   - local: '/scripts/ci/jobs/build-linux.yml'
   - local: '/scripts/ci/jobs/build-linux-petsc.yml'
   - local: '/scripts/ci/jobs/build-linux-frontend.yml'

scripts/ci/jobs/build-docs.yml

@@ -58,7 +58,7 @@ check docs links:
   tags: [ docker ]
   allow_failure: true
   image: $WEB_IMAGE
-  needs: [meta, "build docs"]
+  needs: [meta, ci_images, "build docs"]
   extends:
       - .rules-master-manual
   script:

scripts/ci/jobs/build-gui-linux.yml

@@ -4,7 +4,7 @@ build gui linux:
     - .test-artifacts
   stage: build
   image: $CONTAINER_GCC_GUI_IMAGE
-  needs: [meta, "pre commit"]
+  needs: [meta, ci_images]
   tags: [ docker ]
   rules:
     - if: '$CI_COMMIT_BRANCH == "master"'

scripts/ci/jobs/build-linux.yml

@@ -4,7 +4,7 @@ build linux:
     - .template-build-linux
     - .test-artifacts
   tags: [ docker ]
-  needs: [meta, "pre commit"]
+  needs: [meta, ci_images]
   timeout: 2h
   variables:
     BUILD_CTEST_LARGE_ON_MASTER: "true"
@@ -24,7 +24,7 @@ linux ctest large:
   extends:
     - .template-build-linux
   tags: [ docker, envinf2 ]
-  needs: [meta, "pre commit"]
+  needs: [meta, ci_images]
   rules:
     - when: manual
       allow_failure: true
@@ -44,7 +44,7 @@ build linux (no unity):
     - .template-build-linux
     - .test-artifacts
   tags: [ docker ]
-  needs: [meta, "pre commit"]
+  needs: [meta, ci_images]
   timeout: 1h
   variables:
     BUILD_DIR: "no-unity"
@@ -63,7 +63,7 @@ build linux (no deps, no procs):
   image: $CONTAINER_GCC_IMAGE
   extends: .template-build-linux
   tags: [ docker ]
-  needs: [meta, "pre commit"]
+  needs: [ meta, ci_images ]
   timeout: 1h
   variables:
     BUILD_TESTS: "false"

scripts/ci/jobs/check-header.yml

@@ -3,7 +3,7 @@ check header:
   allow_failure: true
   extends:
     - .rules-master-manual
-  needs: [meta]
+  needs: [ meta, ci_images ]
   variables:
     BUILD_DIR: "../build/check-header"
     CMAKE_ARGS: "-DOGS_CHECK_HEADER_COMPILATION=ON -DOGS_BUILD_GUI=ON -DBUILD_SHARED_LIBS=ON"

scripts/ci/jobs/build_image.yml → scripts/ci/jobs/ci_images.yml

-build container images:
+ci_images:
   stage: .pre
+  variables:
+    CONTAINER_REGISTRY: ${CI_REGISTRY}/${CI_PROJECT_PATH}
+    CONTAINER_TAG: latest
   rules:
-    - if: $CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/
-      when: never
-    - if: '$CI_PROJECT_NAMESPACE == "endjunction" || $CI_PROJECT_NAMESPACE == "bilke" || $CI_PROJECT_NAMESPACE == "ogs"'
+    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
       changes:
       - ThirdParty/container-maker
       - scripts/ci/jobs/build_image.yml
+      - scripts/docker/saxon/**/*
+      - scripts/docker/Dockerfile.pre-commit
+      - scripts/docker/Dockerfile.web
+      variables:
+        BUILD_CONTAINER: "true"
+        CONTAINER_TAG: ${CI_COMMIT_REF_SLUG}
+    - when: always
   tags: [envinf1-shell]
-  extends: .container-maker-setup
+  artifacts:
+    reports:
+      dotenv: build.env
   script:
+    - |
+      echo "CONTAINER_GCC_IMAGE=${CONTAINER_REGISTRY}/gcc:${CONTAINER_TAG}" >> build.env
+      echo "CONTAINER_GCC_GUI_IMAGE=${CONTAINER_REGISTRY}/gcc-gui:${CONTAINER_TAG}" >> build.env
+      echo "CONTAINER_CLANG_IMAGE=${CONTAINER_REGISTRY}/clang:${CONTAINER_TAG}" >> build.env
+      echo "XUNIT_TO_JUNIT_IMAGE=${CONTAINER_REGISTRY}/xunit-to-junit:${CONTAINER_TAG}" >> build.env
+      echo "PRECOMMIT_IMAGE=${CONTAINER_REGISTRY}/pre-commit:${CONTAINER_TAG}" >> build.env
+      echo "WEB_IMAGE=${CONTAINER_REGISTRY}/web:${CONTAINER_TAG}" >> build.env
+      cat build.env
+      if [ "${BUILD_CONTAINER}" != true ]; then exit 0; fi
+    - source build.env
+    - !reference [.container-maker-setup, before_script]
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - poetry run ogscm compiler.py ogs.py --build --ogs off
       --pm system --cvode --cppcheck --docs --gcovr --mfront --ccache
@@ -23,53 +44,10 @@ build container images:
       --pm conan --compiler clang --compiler_version 9 --ccache
       --version_file ../../web/data/versions.json
       --tag $CONTAINER_CLANG_IMAGE --upload
-
-### global project images ###
-build xunit-to-junit image:
-  stage: .pre
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/
-      when: never
-    - if: '$CI_PROJECT_NAMESPACE == "bilke" || $CI_PROJECT_NAMESPACE == "ogs"'
-      changes:
-      - scripts/docker/saxon/**/*
-      - scripts/ci/jobs/build_image.yml
-  tags:
-    - envinf1-shell
-  script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - cd ${CI_PROJECT_DIR}
     - docker build --cache-from $XUNIT_TO_JUNIT_IMAGE -t $XUNIT_TO_JUNIT_IMAGE -f scripts/docker/saxon/Dockerfile scripts/docker/saxon
     - docker push $XUNIT_TO_JUNIT_IMAGE
-
-
-build pre-commit image:
-  stage: .pre
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/
-      when: never
-    - if: '$CI_PROJECT_NAMESPACE == "bilke" || $CI_PROJECT_NAMESPACE == "ogs"'
-      changes:
-      - scripts/docker/Dockerfile.pre-commit
-      - scripts/ci/jobs/build_image.yml
-  tags:
-    - envinf1-shell
-  script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - docker build --cache-from $PRECOMMIT_IMAGE -t $PRECOMMIT_IMAGE -f scripts/docker/Dockerfile.pre-commit scripts/docker
     - docker push $PRECOMMIT_IMAGE
-
-build web image:
-  stage: .pre
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/
-      when: never
-    - if: '$CI_PROJECT_NAMESPACE == "bilke" || $CI_PROJECT_NAMESPACE == "ogs"'
-      changes:
-      - scripts/docker/Dockerfile.web
-      - scripts/ci/jobs/build_image.yml
-  tags:
-    - envinf1-shell
-  script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - docker build --cache-from $WEB_IMAGE -t $WEB_IMAGE -f scripts/docker/Dockerfile.web scripts/docker
     - docker push $WEB_IMAGE

scripts/ci/jobs/clang-sanitizer.yml

@@ -2,14 +2,13 @@ clang sanitizer:
   stage: check
   extends:
     - .rules-manual
-  needs: ["pre commit"]
+  needs: [ ci_images ]
   variables:
     BUILD_DIR: "../build/sanitizer"
     CMAKE_ARGS: "-DOGS_ADDRESS_SANITIZER=ON -DOGS_UNDEFINED_BEHAVIOR_SANITIZER=ON"
     UBSAN_OPTIONS: "print_stacktrace=1"
     LSAN_OPTIONS: "suppressions=$CI_PROJECT_DIR/scripts/test/leak_sanitizer.suppressions"
-  image:
-    name: $CONTAINER_CLANG_IMAGE
+  image: $CONTAINER_CLANG_IMAGE
   script:
     - rm -rf $BUILD_DIR
     - mkdir -p $BUILD_DIR

scripts/ci/jobs/clang-tidy.yml

@@ -2,12 +2,11 @@ clang tidy:
   stage: check
   extends:
     - .rules-manual
-  needs: ["pre commit"]
+  needs: [ ci_images ]
   variables:
     BUILD_DIR: "../build/tidy"
     CMAKE_ARGS: "-DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF -DCMAKE_CXX_CLANG_TIDY=clang-tidy-9"
-  image:
-    name: $CONTAINER_CLANG_IMAGE
+  image: $CONTAINER_CLANG_IMAGE
   script:
     - rm -rf $BUILD_DIR
     - mkdir -p $BUILD_DIR

scripts/ci/jobs/code-quality.yml

 cppcheck:
   stage: check
   image: $CONTAINER_GCC_IMAGE
-  needs: ["pre commit", meta]
+  needs: [ ci_images, meta ]
   before_script:
     - mkdir -p build
     - cd build

scripts/ci/jobs/meta.yml

 meta:
   stage: .pre
   tags: [shell]
-  rules:
-    - if: '$CI_PROJECT_NAMESPACE != "ogs"'
-      changes:
-      - ThirdParty/container-maker
-      - scripts/ci/jobs/build_image.yml
-      variables:
-        OVERWRITE_CONTAINER: "true"
-    - when: always
   variables:
     GIT_DEPTH: 1000
   script:
     - echo "OGS_VERSION=${CI_COMMIT_TAG:-`git describe --tags --long --dirty --always`}" >> build.env
-    - |
-      if [ "$OVERWRITE_CONTAINER" == "true" ]; then
-          echo "CONTAINER_GCC_IMAGE=$CI_REGISTRY_IMAGE/gcc:$CI_COMMIT_REF_SLUG" >> build.env
-          echo "CONTAINER_GCC_GUI_IMAGE=$CI_REGISTRY_IMAGE/gcc-gui:$CI_COMMIT_REF_SLUG" >> build.env
-          echo "CONTAINER_CLANG_IMAGE=$CI_REGISTRY_IMAGE/clang:$CI_COMMIT_REF_SLUG" >> build.env
-      fi
     - cat build.env
   artifacts:
     reports:

scripts/ci/jobs/pre-commit.yml

 pre commit:
   stage: preparation
   image: $PRECOMMIT_IMAGE
+  needs: [ ci_images ]
   script:
     - pre-commit install
     - pre-commit run --all-files

scripts/ci/jobs/web.yml

@@ -4,7 +4,7 @@ web url checker:
   tags: [ docker ]
   extends:
     - .rules-manual
-  needs: ["pre commit"]
+  needs: [ ci_images ]
   image: $WEB_IMAGE
   script:
     - cd web
@@ -18,7 +18,7 @@ preview web site:
   stage: build
   tags: [ docker ]
   image: $WEB_IMAGE
-  needs: ["pre commit"]
+  needs: [ ci_images ]
   script:
     # HACK to easier linking to the generated pages
     - echo '<meta http-equiv="REFRESH" content="0;URL=web/public/index.html">' >> WebPreview.html
@@ -43,7 +43,7 @@ preview web site:
 deploy web site:
   stage: package
   image: $WEB_IMAGE
-  needs: ["pre commit"]
+  needs: [ ci_images ]
   rules:
     - if: $CI_COMMIT_TAG
     - if: '$CI_COMMIT_BRANCH == "master"'