From 1c97e42017990f4ceed1fdad001aeb692b774b32 Mon Sep 17 00:00:00 2001
From: Lars Bilke <lars.bilke@ufz.de>
Date: Tue, 25 May 2021 09:18:35 +0200
Subject: [PATCH] [ci] Streamlined Docker image selection logic.
---
.gitlab-ci.yml | 18 +----
scripts/ci/jobs/build-docs.yml | 2 +-
scripts/ci/jobs/build-gui-linux.yml | 2 +-
scripts/ci/jobs/build-linux.yml | 8 +-
scripts/ci/jobs/check-header.yml | 2 +-
.../jobs/{build_image.yml => ci_images.yml} | 76 +++++++------------
scripts/ci/jobs/clang-sanitizer.yml | 5 +-
scripts/ci/jobs/clang-tidy.yml | 5 +-
scripts/ci/jobs/code-quality.yml | 2 +-
scripts/ci/jobs/meta.yml | 14 ----
scripts/ci/jobs/pre-commit.yml | 1 +
scripts/ci/jobs/web.yml | 6 +-
12 files changed, 45 insertions(+), 96 deletions(-)
rename scripts/ci/jobs/{build_image.yml => ci_images.yml} (51%)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 524407119bc..247ac261971 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,13 +6,6 @@ stages:
- package
variables:
- # Docker image names
- CONTAINER_GCC_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/gcc
- CONTAINER_GCC_GUI_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/gcc-gui
- CONTAINER_CLANG_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/clang
- XUNIT_TO_JUNIT_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/xunit-to-junit
- PRECOMMIT_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/pre-commit
- WEB_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/web
# Build config
BUILD_TYPE: Release
BUILD_PROCESSES: "" # Empty string: builds all processes
@@ -32,18 +25,11 @@ workflow:
- if: $CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/ # release branches, e.g. v6.x.x
include:
- # extends
- - local: '/scripts/ci/extends/defaults.yml'
- - local: '/scripts/ci/extends/rules.yml'
- - local: '/scripts/ci/extends/template-build-linux.yml'
- - local: '/scripts/ci/extends/template-build-win.yml'
- - local: '/scripts/ci/extends/container-maker-setup.yml'
- - local: '/scripts/ci/extends/vs2019-environment.yml'
- - local: '/scripts/ci/extends/test-artifacts.yml'
+ - local: 'scripts/ci/extends/*.yml'
- local: '/scripts/ci/jobs/meta.yml'
+ - local: '/scripts/ci/jobs/ci_images.yml'
- local: '/scripts/ci/jobs/pre-commit.yml'
# jobs, can be indiviually disabled for testing
- - local: '/scripts/ci/jobs/build_image.yml'
- local: '/scripts/ci/jobs/build-linux.yml'
- local: '/scripts/ci/jobs/build-linux-petsc.yml'
- local: '/scripts/ci/jobs/build-linux-frontend.yml'
diff --git a/scripts/ci/jobs/build-docs.yml b/scripts/ci/jobs/build-docs.yml
index 47c1d4c24d3..bb7edcbfacd 100644
--- a/scripts/ci/jobs/build-docs.yml
+++ b/scripts/ci/jobs/build-docs.yml
@@ -58,7 +58,7 @@ check docs links:
tags: [ docker ]
allow_failure: true
image: $WEB_IMAGE
- needs: [meta, "build docs"]
+ needs: [meta, ci_images, "build docs"]
extends:
- .rules-master-manual
script:
diff --git a/scripts/ci/jobs/build-gui-linux.yml b/scripts/ci/jobs/build-gui-linux.yml
index f08916eedd9..36afc41b47b 100644
--- a/scripts/ci/jobs/build-gui-linux.yml
+++ b/scripts/ci/jobs/build-gui-linux.yml
@@ -4,7 +4,7 @@ build gui linux:
- .test-artifacts
stage: build
image: $CONTAINER_GCC_GUI_IMAGE
- needs: [meta, "pre commit"]
+ needs: [meta, ci_images]
tags: [ docker ]
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
diff --git a/scripts/ci/jobs/build-linux.yml b/scripts/ci/jobs/build-linux.yml
index 3dd53aa4bba..13fb32e26f2 100644
--- a/scripts/ci/jobs/build-linux.yml
+++ b/scripts/ci/jobs/build-linux.yml
@@ -4,7 +4,7 @@ build linux:
- .template-build-linux
- .test-artifacts
tags: [ docker ]
- needs: [meta, "pre commit"]
+ needs: [meta, ci_images]
timeout: 2h
variables:
BUILD_CTEST_LARGE_ON_MASTER: "true"
@@ -24,7 +24,7 @@ linux ctest large:
extends:
- .template-build-linux
tags: [ docker, envinf2 ]
- needs: [meta, "pre commit"]
+ needs: [meta, ci_images]
rules:
- when: manual
allow_failure: true
@@ -44,7 +44,7 @@ build linux (no unity):
- .template-build-linux
- .test-artifacts
tags: [ docker ]
- needs: [meta, "pre commit"]
+ needs: [meta, ci_images]
timeout: 1h
variables:
BUILD_DIR: "no-unity"
@@ -63,7 +63,7 @@ build linux (no deps, no procs):
image: $CONTAINER_GCC_IMAGE
extends: .template-build-linux
tags: [ docker ]
- needs: [meta, "pre commit"]
+ needs: [ meta, ci_images ]
timeout: 1h
variables:
BUILD_TESTS: "false"
diff --git a/scripts/ci/jobs/check-header.yml b/scripts/ci/jobs/check-header.yml
index ef81d2064d3..acdf2629183 100644
--- a/scripts/ci/jobs/check-header.yml
+++ b/scripts/ci/jobs/check-header.yml
@@ -3,7 +3,7 @@ check header:
allow_failure: true
extends:
- .rules-master-manual
- needs: [meta]
+ needs: [ meta, ci_images ]
variables:
BUILD_DIR: "../build/check-header"
CMAKE_ARGS: "-DOGS_CHECK_HEADER_COMPILATION=ON -DOGS_BUILD_GUI=ON -DBUILD_SHARED_LIBS=ON"
diff --git a/scripts/ci/jobs/build_image.yml b/scripts/ci/jobs/ci_images.yml
similarity index 51%
rename from scripts/ci/jobs/build_image.yml
rename to scripts/ci/jobs/ci_images.yml
index f3da3154023..f152cb2a4be 100644
--- a/scripts/ci/jobs/build_image.yml
+++ b/scripts/ci/jobs/ci_images.yml
@@ -1,15 +1,36 @@
-build container images:
+ci_images:
stage: .pre
+ variables:
+ CONTAINER_REGISTRY: ${CI_REGISTRY}/${CI_PROJECT_PATH}
+ CONTAINER_TAG: latest
rules:
- - if: $CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/
- when: never
- - if: '$CI_PROJECT_NAMESPACE == "endjunction" || $CI_PROJECT_NAMESPACE == "bilke" || $CI_PROJECT_NAMESPACE == "ogs"'
+ - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
changes:
- ThirdParty/container-maker
- scripts/ci/jobs/build_image.yml
+ - scripts/docker/saxon/**/*
+ - scripts/docker/Dockerfile.pre-commit
+ - scripts/docker/Dockerfile.web
+ variables:
+ BUILD_CONTAINER: "true"
+ CONTAINER_TAG: ${CI_COMMIT_REF_SLUG}
+ - when: always
tags: [envinf1-shell]
- extends: .container-maker-setup
+ artifacts:
+ reports:
+ dotenv: build.env
script:
+ - |
+ echo "CONTAINER_GCC_IMAGE=${CONTAINER_REGISTRY}/gcc:${CONTAINER_TAG}" >> build.env
+ echo "CONTAINER_GCC_GUI_IMAGE=${CONTAINER_REGISTRY}/gcc-gui:${CONTAINER_TAG}" >> build.env
+ echo "CONTAINER_CLANG_IMAGE=${CONTAINER_REGISTRY}/clang:${CONTAINER_TAG}" >> build.env
+ echo "XUNIT_TO_JUNIT_IMAGE=${CONTAINER_REGISTRY}/xunit-to-junit:${CONTAINER_TAG}" >> build.env
+ echo "PRECOMMIT_IMAGE=${CONTAINER_REGISTRY}/pre-commit:${CONTAINER_TAG}" >> build.env
+ echo "WEB_IMAGE=${CONTAINER_REGISTRY}/web:${CONTAINER_TAG}" >> build.env
+ cat build.env
+ if [ "${BUILD_CONTAINER}" != true ]; then exit 0; fi
+ - source build.env
+ - !reference [.container-maker-setup, before_script]
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- poetry run ogscm compiler.py ogs.py --build --ogs off
--pm system --cvode --cppcheck --docs --gcovr --mfront --ccache
@@ -23,53 +44,10 @@ build container images:
--pm conan --compiler clang --compiler_version 9 --ccache
--version_file ../../web/data/versions.json
--tag $CONTAINER_CLANG_IMAGE --upload
-
-### global project images ###
-build xunit-to-junit image:
- stage: .pre
- rules:
- - if: $CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/
- when: never
- - if: '$CI_PROJECT_NAMESPACE == "bilke" || $CI_PROJECT_NAMESPACE == "ogs"'
- changes:
- - scripts/docker/saxon/**/*
- - scripts/ci/jobs/build_image.yml
- tags:
- - envinf1-shell
- script:
- - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ - cd ${CI_PROJECT_DIR}
- docker build --cache-from $XUNIT_TO_JUNIT_IMAGE -t $XUNIT_TO_JUNIT_IMAGE -f scripts/docker/saxon/Dockerfile scripts/docker/saxon
- docker push $XUNIT_TO_JUNIT_IMAGE
-
-
-build pre-commit image:
- stage: .pre
- rules:
- - if: $CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/
- when: never
- - if: '$CI_PROJECT_NAMESPACE == "bilke" || $CI_PROJECT_NAMESPACE == "ogs"'
- changes:
- - scripts/docker/Dockerfile.pre-commit
- - scripts/ci/jobs/build_image.yml
- tags:
- - envinf1-shell
- script:
- - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker build --cache-from $PRECOMMIT_IMAGE -t $PRECOMMIT_IMAGE -f scripts/docker/Dockerfile.pre-commit scripts/docker
- docker push $PRECOMMIT_IMAGE
-
-build web image:
- stage: .pre
- rules:
- - if: $CI_COMMIT_BRANCH =~ /^v[0-9]\.[0-9]\.[0-9]/
- when: never
- - if: '$CI_PROJECT_NAMESPACE == "bilke" || $CI_PROJECT_NAMESPACE == "ogs"'
- changes:
- - scripts/docker/Dockerfile.web
- - scripts/ci/jobs/build_image.yml
- tags:
- - envinf1-shell
- script:
- - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker build --cache-from $WEB_IMAGE -t $WEB_IMAGE -f scripts/docker/Dockerfile.web scripts/docker
- docker push $WEB_IMAGE
diff --git a/scripts/ci/jobs/clang-sanitizer.yml b/scripts/ci/jobs/clang-sanitizer.yml
index 78ad8a1f464..52055dcd13c 100644
--- a/scripts/ci/jobs/clang-sanitizer.yml
+++ b/scripts/ci/jobs/clang-sanitizer.yml
@@ -2,14 +2,13 @@ clang sanitizer:
stage: check
extends:
- .rules-manual
- needs: ["pre commit"]
+ needs: [ ci_images ]
variables:
BUILD_DIR: "../build/sanitizer"
CMAKE_ARGS: "-DOGS_ADDRESS_SANITIZER=ON -DOGS_UNDEFINED_BEHAVIOR_SANITIZER=ON"
UBSAN_OPTIONS: "print_stacktrace=1"
LSAN_OPTIONS: "suppressions=$CI_PROJECT_DIR/scripts/test/leak_sanitizer.suppressions"
- image:
- name: $CONTAINER_CLANG_IMAGE
+ image: $CONTAINER_CLANG_IMAGE
script:
- rm -rf $BUILD_DIR
- mkdir -p $BUILD_DIR
diff --git a/scripts/ci/jobs/clang-tidy.yml b/scripts/ci/jobs/clang-tidy.yml
index 2a2c0f14a12..2a44565fdf5 100644
--- a/scripts/ci/jobs/clang-tidy.yml
+++ b/scripts/ci/jobs/clang-tidy.yml
@@ -2,12 +2,11 @@ clang tidy:
stage: check
extends:
- .rules-manual
- needs: ["pre commit"]
+ needs: [ ci_images ]
variables:
BUILD_DIR: "../build/tidy"
CMAKE_ARGS: "-DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF -DCMAKE_CXX_CLANG_TIDY=clang-tidy-9"
- image:
- name: $CONTAINER_CLANG_IMAGE
+ image: $CONTAINER_CLANG_IMAGE
script:
- rm -rf $BUILD_DIR
- mkdir -p $BUILD_DIR
diff --git a/scripts/ci/jobs/code-quality.yml b/scripts/ci/jobs/code-quality.yml
index a44664c5369..6967592e56f 100644
--- a/scripts/ci/jobs/code-quality.yml
+++ b/scripts/ci/jobs/code-quality.yml
@@ -1,7 +1,7 @@
cppcheck:
stage: check
image: $CONTAINER_GCC_IMAGE
- needs: ["pre commit", meta]
+ needs: [ ci_images, meta ]
before_script:
- mkdir -p build
- cd build
diff --git a/scripts/ci/jobs/meta.yml b/scripts/ci/jobs/meta.yml
index b82735b9583..c0c9ed1adf9 100644
--- a/scripts/ci/jobs/meta.yml
+++ b/scripts/ci/jobs/meta.yml
@@ -1,24 +1,10 @@
meta:
stage: .pre
tags: [shell]
- rules:
- - if: '$CI_PROJECT_NAMESPACE != "ogs"'
- changes:
- - ThirdParty/container-maker
- - scripts/ci/jobs/build_image.yml
- variables:
- OVERWRITE_CONTAINER: "true"
- - when: always
variables:
GIT_DEPTH: 1000
script:
- echo "OGS_VERSION=${CI_COMMIT_TAG:-`git describe --tags --long --dirty --always`}" >> build.env
- - |
- if [ "$OVERWRITE_CONTAINER" == "true" ]; then
- echo "CONTAINER_GCC_IMAGE=$CI_REGISTRY_IMAGE/gcc:$CI_COMMIT_REF_SLUG" >> build.env
- echo "CONTAINER_GCC_GUI_IMAGE=$CI_REGISTRY_IMAGE/gcc-gui:$CI_COMMIT_REF_SLUG" >> build.env
- echo "CONTAINER_CLANG_IMAGE=$CI_REGISTRY_IMAGE/clang:$CI_COMMIT_REF_SLUG" >> build.env
- fi
- cat build.env
artifacts:
reports:
diff --git a/scripts/ci/jobs/pre-commit.yml b/scripts/ci/jobs/pre-commit.yml
index 819b0b73f5d..b2a8378ccdd 100644
--- a/scripts/ci/jobs/pre-commit.yml
+++ b/scripts/ci/jobs/pre-commit.yml
@@ -1,6 +1,7 @@
pre commit:
stage: preparation
image: $PRECOMMIT_IMAGE
+ needs: [ ci_images ]
script:
- pre-commit install
- pre-commit run --all-files
diff --git a/scripts/ci/jobs/web.yml b/scripts/ci/jobs/web.yml
index 122810f08ba..fe23de056c2 100644
--- a/scripts/ci/jobs/web.yml
+++ b/scripts/ci/jobs/web.yml
@@ -4,7 +4,7 @@ web url checker:
tags: [ docker ]
extends:
- .rules-manual
- needs: ["pre commit"]
+ needs: [ ci_images ]
image: $WEB_IMAGE
script:
- cd web
@@ -18,7 +18,7 @@ preview web site:
stage: build
tags: [ docker ]
image: $WEB_IMAGE
- needs: ["pre commit"]
+ needs: [ ci_images ]
script:
# HACK to easier linking to the generated pages
- echo '<meta http-equiv="REFRESH" content="0;URL=web/public/index.html">' >> WebPreview.html
@@ -43,7 +43,7 @@ preview web site:
deploy web site:
stage: package
image: $WEB_IMAGE
- needs: ["pre commit"]
+ needs: [ ci_images ]
rules:
- if: $CI_COMMIT_TAG
- if: '$CI_COMMIT_BRANCH == "master"'
--
GitLab