From ff695bd7da3021f1a3be51f5cc50cbce5244f436 Mon Sep 17 00:00:00 2001
From: Lars Bilke <lars.bilke@ufz.de>
Date: Fri, 3 Jul 2020 09:55:12 +0200
Subject: [PATCH] [ci] Docker image generation for MRs.
---
.gitlab-ci.yml | 3 +-
.../ci/extends/template-build-dockerfile.yml | 2 -
scripts/ci/extends/template-build-linux.yml | 2 +-
scripts/ci/extends/template-build-win.yml | 1 +
scripts/ci/jobs/build-docs.yml | 11 ++--
scripts/ci/jobs/build-gui-linux.yml | 4 --
scripts/ci/jobs/build-gui-mac.yml | 4 --
scripts/ci/jobs/build-gui-win.yml | 4 --
scripts/ci/jobs/build-linux-frontend.yml | 1 -
scripts/ci/jobs/build-linux-petsc.yml | 1 -
scripts/ci/jobs/build-linux.yml | 1 -
scripts/ci/jobs/build-mac.yml | 1 -
scripts/ci/jobs/build-win.yml | 1 -
scripts/ci/jobs/build_image.yml | 54 ++++++-------------
scripts/ci/jobs/check-header.yml | 6 +--
scripts/ci/jobs/checks.yml | 1 -
scripts/ci/jobs/clang-sanitizer.yml | 7 +--
scripts/ci/jobs/clang-tidy.yml | 5 +-
scripts/ci/jobs/code-quality.yml | 3 +-
scripts/ci/jobs/container.yml | 1 -
scripts/ci/jobs/meta.yml | 20 ++++++-
scripts/ci/jobs/pre-commit.yml | 5 +-
scripts/ci/jobs/tests-large.yml | 7 +--
scripts/ci/jobs/web.yml | 7 ++-
24 files changed, 51 insertions(+), 101 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 920bd375c4d..29bf2c3476b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,5 +1,5 @@
stages:
- - pre-check
+ - preparation
- build
- check
- gui
@@ -11,7 +11,6 @@ variables:
CONTAINER_GCC_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/gcc
CONTAINER_GCC_GUI_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/gcc-gui
CONTAINER_CLANG_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/clang
- CONTAINER_TAG: latest
XUNIT_TO_JUNIT_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/xunit-to-junit
PRECOMMIT_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/pre-commit
WEB_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/web
diff --git a/scripts/ci/extends/template-build-dockerfile.yml b/scripts/ci/extends/template-build-dockerfile.yml
index 8a8404325f9..252ac45e173 100644
--- a/scripts/ci/extends/template-build-dockerfile.yml
+++ b/scripts/ci/extends/template-build-dockerfile.yml
@@ -15,8 +15,6 @@
- scripts/ci/extends/template-build-dockerfile.yml
tags:
- docker-shell
- variables:
- GIT_LFS_SKIP_SMUDGE: "1"
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker pull $IMAGE_REF || true
diff --git a/scripts/ci/extends/template-build-linux.yml b/scripts/ci/extends/template-build-linux.yml
index 995cb4b4369..dd4594d4b72 100644
--- a/scripts/ci/extends/template-build-linux.yml
+++ b/scripts/ci/extends/template-build-linux.yml
@@ -4,7 +4,7 @@
- docker
extends:
- .test-artifacts
-
+ dependencies: [meta, meta_container]
variables:
CCACHE_DIR: "$CI_PROJECT_DIR/.ccache"
diff --git a/scripts/ci/extends/template-build-win.yml b/scripts/ci/extends/template-build-win.yml
index 35e5837d93d..7cb85ec78b1 100644
--- a/scripts/ci/extends/template-build-win.yml
+++ b/scripts/ci/extends/template-build-win.yml
@@ -4,6 +4,7 @@
- windows
extends:
- .vs2019-environment
+ dependencies: [meta, meta_container]
variables:
script:
- mkdir -p $env:BUILD_DIR
diff --git a/scripts/ci/jobs/build-docs.yml b/scripts/ci/jobs/build-docs.yml
index f19bf986ee5..9639f63c981 100644
--- a/scripts/ci/jobs/build-docs.yml
+++ b/scripts/ci/jobs/build-docs.yml
@@ -2,18 +2,15 @@ build docs:
stage: build
image:
name: $CONTAINER_GCC_IMAGE
- needs: [meta]
-
+ dependencies: [meta, meta_container]
before_script:
# HACK to easier linking to the generated pages
- echo '<meta http-equiv="REFRESH" content="0;URL=build/docs/index.html">' >> Doxygen.html
- mkdir -p build
- cd build
-
script:
- cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DOGS_USE_CONAN=OFF -DOGS_BUILD_PROCESSES=GroundwaterFlow
- cmake --build . --target doc > >(tee make-docs.output)
-
artifacts:
expose_as: 'Doxygen'
paths:
@@ -24,8 +21,7 @@ build docs:
check docs links:
stage: check
image: $WEB_IMAGE
- needs:
- - job: build docs
+ dependencies: [meta_container, "build docs"]
extends:
- .rules-master-manual
script:
@@ -42,8 +38,7 @@ deploy doxygen:
image: $WEB_IMAGE
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
- needs:
- - job: build docs
+ dependencies: [meta_container, "build docs"]
variables:
GIT_STRATEGY: none
script:
diff --git a/scripts/ci/jobs/build-gui-linux.yml b/scripts/ci/jobs/build-gui-linux.yml
index efc4b9e86ff..be06bc9f320 100644
--- a/scripts/ci/jobs/build-gui-linux.yml
+++ b/scripts/ci/jobs/build-gui-linux.yml
@@ -6,10 +6,6 @@ build gui linux:
tags:
- envinf1
- docker
- needs:
- - meta
- - job: build linux
- artifacts: false
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
- changes:
diff --git a/scripts/ci/jobs/build-gui-mac.yml b/scripts/ci/jobs/build-gui-mac.yml
index d5f81bccdc8..6d519473c7c 100644
--- a/scripts/ci/jobs/build-gui-mac.yml
+++ b/scripts/ci/jobs/build-gui-mac.yml
@@ -1,10 +1,6 @@
build gui mac:
extends: .template-build-linux
stage: gui
- needs:
- - meta
- - job: build mac
- artifacts: false
tags:
- mac
rules:
diff --git a/scripts/ci/jobs/build-gui-win.yml b/scripts/ci/jobs/build-gui-win.yml
index 12d32cd0b09..13d94a8b8de 100644
--- a/scripts/ci/jobs/build-gui-win.yml
+++ b/scripts/ci/jobs/build-gui-win.yml
@@ -1,10 +1,6 @@
build gui win:
extends: .template-build-win
stage: gui
- needs:
- - meta
- - job: build win
- artifacts: false
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
- changes:
diff --git a/scripts/ci/jobs/build-linux-frontend.yml b/scripts/ci/jobs/build-linux-frontend.yml
index 9cdd3a5fb30..9532d91bff2 100644
--- a/scripts/ci/jobs/build-linux-frontend.yml
+++ b/scripts/ci/jobs/build-linux-frontend.yml
@@ -4,7 +4,6 @@ build linux frontend:
- .rules-master-manual
tags:
- frontend2
- needs: [meta]
variables:
BUILD_DIR: "build"
CCACHE_DIR: "/data/ogs/.ccache"
diff --git a/scripts/ci/jobs/build-linux-petsc.yml b/scripts/ci/jobs/build-linux-petsc.yml
index 768514dae81..41f6a75142b 100644
--- a/scripts/ci/jobs/build-linux-petsc.yml
+++ b/scripts/ci/jobs/build-linux-petsc.yml
@@ -2,7 +2,6 @@ build linux petsc:
extends: .template-build-linux
tags:
- envinf1-shell
- needs: [meta]
variables:
BUILD_DIR: "build-petsc"
CMAKE_ARGS: "-DOGS_USE_CONAN=OFF -DOGS_USE_PETSC=ON -DBUILD_SHARED_LIBS=ON"
diff --git a/scripts/ci/jobs/build-linux.yml b/scripts/ci/jobs/build-linux.yml
index 2fb29d135b8..702e3597195 100644
--- a/scripts/ci/jobs/build-linux.yml
+++ b/scripts/ci/jobs/build-linux.yml
@@ -11,4 +11,3 @@ build linux:
-DOGS_USE_PYTHON=ON
image:
name: $CONTAINER_GCC_IMAGE
- needs: [meta]
diff --git a/scripts/ci/jobs/build-mac.yml b/scripts/ci/jobs/build-mac.yml
index 0632b6634fd..3ede1d8ec3c 100644
--- a/scripts/ci/jobs/build-mac.yml
+++ b/scripts/ci/jobs/build-mac.yml
@@ -2,7 +2,6 @@ build mac:
extends: .template-build-linux
tags:
- mac
- needs: [meta]
variables:
BUILD_DIR: "build-mac"
CMAKE_ARGS: >-
diff --git a/scripts/ci/jobs/build-win.yml b/scripts/ci/jobs/build-win.yml
index 1f3212e46ff..1a2b931cb64 100644
--- a/scripts/ci/jobs/build-win.yml
+++ b/scripts/ci/jobs/build-win.yml
@@ -1,6 +1,5 @@
build win:
extends: .template-build-win
- needs: [meta]
variables:
BUILD_DIR: build-win
CMAKE_ARGS: >-
diff --git a/scripts/ci/jobs/build_image.yml b/scripts/ci/jobs/build_image.yml
index e1b4c67b3b9..498d47c4b7e 100644
--- a/scripts/ci/jobs/build_image.yml
+++ b/scripts/ci/jobs/build_image.yml
@@ -1,33 +1,23 @@
build container images:
- stage: .pre
+ stage: preparation
rules:
- # TODO
- # - changes:
- # - ThirdParty/container-maker
- # - scripts/ci/jobs/build_image.yml
- # when: always
- - if: '$CI_COMMIT_BRANCH == "master"'
- when: always
- tags:
- - envinf1-shell # can be changed back when instance-level vars are implemented
-
- variables:
- GIT_LFS_SKIP_SMUDGE: "1"
+ - changes:
+ - ThirdParty/container-maker
+ - scripts/ci/jobs/build_image.yml
+ dependencies: [meta_container]
+ tags: [docker-shell]
extends: .container-maker-setup
script:
- - docker login -u $OGS_CI_REGISTRY_USER -p $OGS_CI_REGISTRY_PASSWORD $CI_REGISTRY
- - docker pull registry.opengeosys.org/ogs/ogs/ogs/gcc || true
+ - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- python ThirdParty/container-maker/ogscm/cli.py --build --ogs off
- --pm system --cvode --cppcheck --docs --gcovr
- --tag $CONTAINER_GCC_IMAGE:$CONTAINER_TAG --upload
- - docker pull registry.opengeosys.org/ogs/ogs/ogs/gcc-gui || true
+ --pm system --cvode --cppcheck --docs --gcovr --tfel
+ --tag $CONTAINER_GCC_IMAGE --upload
- python ThirdParty/container-maker/ogscm/cli.py --build --ogs off
--pm system --cvode --cppcheck --docs --gcovr --gui
- --tag $CONTAINER_GCC_GUI_IMAGE:$CONTAINER_TAG --upload
- - docker pull registry.opengeosys.org/ogs/ogs/ogs/clang || true
+ --tag $CONTAINER_GCC_GUI_IMAGE --upload
- python ThirdParty/container-maker/ogscm/cli.py --build --ogs off
--compiler clang --compiler_version 9
- --tag $CONTAINER_CLANG_IMAGE:$CONTAINER_TAG --upload
+ --tag $CONTAINER_CLANG_IMAGE --upload
### global project images ###
build xunit-to-junit image:
@@ -39,14 +29,10 @@ build xunit-to-junit image:
- scripts/ci/jobs/build_image.yml
tags:
- docker-shell
-
- variables:
- GIT_LFS_SKIP_SMUDGE: "1"
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- - docker pull $XUNIT_TO_JUNIT_IMAGE:latest || true
- - docker build --cache-from $XUNIT_TO_JUNIT_IMAGE:latest -t $XUNIT_TO_JUNIT_IMAGE:latest -f scripts/docker/saxon/Dockerfile scripts/docker/saxon
- - docker push $XUNIT_TO_JUNIT_IMAGE:latest
+ - docker build --cache-from $XUNIT_TO_JUNIT_IMAGE -t $XUNIT_TO_JUNIT_IMAGE -f scripts/docker/saxon/Dockerfile scripts/docker/saxon
+ - docker push $XUNIT_TO_JUNIT_IMAGE
build pre-commit image:
@@ -58,13 +44,10 @@ build pre-commit image:
- scripts/ci/jobs/build_image.yml
tags:
- docker-shell
- variables:
- GIT_LFS_SKIP_SMUDGE: "1"
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- - docker pull $PRECOMMIT_IMAGE:latest || true
- - docker build --cache-from $PRECOMMIT_IMAGE:latest -t $PRECOMMIT_IMAGE:latest -f scripts/docker/Dockerfile.pre-commit scripts/docker
- - docker push $PRECOMMIT_IMAGE:latest
+ - docker build --cache-from $PRECOMMIT_IMAGE -t $PRECOMMIT_IMAGE -f scripts/docker/Dockerfile.pre-commit scripts/docker
+ - docker push $PRECOMMIT_IMAGE
build web image:
stage: .pre
@@ -75,10 +58,7 @@ build web image:
- scripts/ci/jobs/build_image.yml
tags:
- docker-shell
- variables:
- GIT_LFS_SKIP_SMUDGE: "1"
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- - docker pull $WEB_IMAGE:latest || true
- - docker build --cache-from $WEB_IMAGE:latest -t $WEB_IMAGE:latest -f scripts/docker/Dockerfile.web scripts/docker
- - docker push $WEB_IMAGE:latest
+ - docker build --cache-from $WEB_IMAGE -t $WEB_IMAGE -f scripts/docker/Dockerfile.web scripts/docker
+ - docker push $WEB_IMAGE
diff --git a/scripts/ci/jobs/check-header.yml b/scripts/ci/jobs/check-header.yml
index c08c0bd2d66..3aecf2df4bc 100644
--- a/scripts/ci/jobs/check-header.yml
+++ b/scripts/ci/jobs/check-header.yml
@@ -4,17 +4,14 @@ check header:
tags:
- docker
extends:
- - .defaults
- .rules-master-manual
-
+ dependencies: [meta_container]
variables:
BUILD_DIR: "build-check-header"
CCACHE_DIR: "$CI_PROJECT_DIR/.ccache"
CMAKE_ARGS: "-DOGS_CHECK_HEADER_COMPILATION=ON -DOGS_BUILD_UTILS=ON -DOGS_BUILD_GUI=ON -DOGS_USE_PYTHON=ON -DBUILD_SHARED_LIBS=ON"
-
image:
name: $CONTAINER_GCC_GUI_IMAGE
-
script:
- mkdir -p $BUILD_DIR
- cd $BUILD_DIR
@@ -24,7 +21,6 @@ check header:
-DCMAKE_BUILD_TYPE=$BUILD_TYPE
-DOGS_BUILD_PROCESSES=$BUILD_PROCESSES || true
- cat CMakeFiles/CMakeError.log
-
cache:
paths:
- $CCACHE_DIR
diff --git a/scripts/ci/jobs/checks.yml b/scripts/ci/jobs/checks.yml
index f5a530d90b9..a9540c2342d 100644
--- a/scripts/ci/jobs/checks.yml
+++ b/scripts/ci/jobs/checks.yml
@@ -17,7 +17,6 @@ compiler warnings:
- "if [[ $(cat make.output | grep warning -i) ]]; then echo 'There were Clang (macOS) compiler warnings:'; cat make.output | grep warning -i; exit_code=1; fi"
- exit $exit_code
-
ctest results:
stage: check
variables:
diff --git a/scripts/ci/jobs/clang-sanitizer.yml b/scripts/ci/jobs/clang-sanitizer.yml
index 03a3d669123..56025a5f2ad 100644
--- a/scripts/ci/jobs/clang-sanitizer.yml
+++ b/scripts/ci/jobs/clang-sanitizer.yml
@@ -1,20 +1,15 @@
clang sanitizer:
stage: check
- tags:
- - docker
extends:
- - .defaults
- .rules-manual
-
+ dependencies: [meta_container]
variables:
BUILD_DIR: "build-sanitizer"
CMAKE_ARGS: "-DOGS_ADDRESS_SANITIZER=ON -DOGS_UNDEFINED_BEHAVIOR_SANITIZER=ON -DOGS_BUILD_UTILS=ON"
UBSAN_OPTIONS: "print_stacktrace=1"
LSAN_OPTIONS: "suppressions=$CI_PROJECT_DIR/scripts/test/leak_sanitizer.suppressions"
-
image:
name: $CONTAINER_CLANG_IMAGE
-
script:
- mkdir -p $BUILD_DIR
- cd $BUILD_DIR
diff --git a/scripts/ci/jobs/clang-tidy.yml b/scripts/ci/jobs/clang-tidy.yml
index bc2a24a0927..ae03354c33a 100644
--- a/scripts/ci/jobs/clang-tidy.yml
+++ b/scripts/ci/jobs/clang-tidy.yml
@@ -1,11 +1,8 @@
clang tidy:
stage: check
- tags:
- - docker
extends:
- - .defaults
- .rules-manual
-
+ dependencies: [meta_container]
variables:
BUILD_DIR: "build-tidy"
CMAKE_ARGS: "-DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF -DCMAKE_CXX_CLANG_TIDY=clang-tidy-9"
diff --git a/scripts/ci/jobs/code-quality.yml b/scripts/ci/jobs/code-quality.yml
index 42a76ea4f4b..35d24fb68df 100644
--- a/scripts/ci/jobs/code-quality.yml
+++ b/scripts/ci/jobs/code-quality.yml
@@ -1,7 +1,6 @@
code_quality:
stage: check
- extends:
- - .defaults
+ needs: []
rules:
variables:
DOCKER_TLS_CERTDIR: "/certs"
diff --git a/scripts/ci/jobs/container.yml b/scripts/ci/jobs/container.yml
index 6a721529610..9a4858443d2 100644
--- a/scripts/ci/jobs/container.yml
+++ b/scripts/ci/jobs/container.yml
@@ -9,7 +9,6 @@ container:
variables:
OPENMPI_VERSIONS: 'off' # 4.0.1
extends:
- - .defaults
- .container-maker-setup
script:
- python ThirdParty/container-maker/ogscm/cli.py -B -C -R --ogs . --pm system --cvode --ompi $OPENMPI_VERSIONS
diff --git a/scripts/ci/jobs/meta.yml b/scripts/ci/jobs/meta.yml
index 213210e4fe1..0abac260b93 100644
--- a/scripts/ci/jobs/meta.yml
+++ b/scripts/ci/jobs/meta.yml
@@ -1,7 +1,7 @@
meta:
stage: .pre
- tags:
- - envinf1-shell
+ tags: [shell]
+ - shell
script:
- git fetch --depth 500
- echo "OGS_VERSION=${CI_COMMIT_TAG:-`git describe --tags --long --dirty --always`}" >> build.env
@@ -9,3 +9,19 @@ meta:
artifacts:
reports:
dotenv: build.env
+
+meta_container:
+ stage: .pre
+ tags: [shell]
+ rules:
+ - changes:
+ - ThirdParty/container-maker
+ - scripts/ci/jobs/build_image.yml
+ script:
+ - echo "CONTAINER_GCC_IMAGE=$CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/gcc:$CI_COMMIT_REF_SLUG" >> build.env
+ - echo "CONTAINER_GCC_GUI_IMAGE=$CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/gcc-gui:$CI_COMMIT_REF_SLUG" >> build.env
+ - echo "CONTAINER_CLANG_IMAGE=$CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/clang:$CI_COMMIT_REF_SLUG" >> build.env
+ - cat build.env
+ artifacts:
+ reports:
+ dotenv: build.env
diff --git a/scripts/ci/jobs/pre-commit.yml b/scripts/ci/jobs/pre-commit.yml
index 7adcf4f9f0e..73d530dfea2 100644
--- a/scripts/ci/jobs/pre-commit.yml
+++ b/scripts/ci/jobs/pre-commit.yml
@@ -1,9 +1,8 @@
pre commit:
- stage: pre-check
+ stage: preparation
image:
name: $PRECOMMIT_IMAGE
- extends:
- - .defaults
+ dependencies: [meta_container]
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
when: never
diff --git a/scripts/ci/jobs/tests-large.yml b/scripts/ci/jobs/tests-large.yml
index 5463bb21fe4..0424a740778 100644
--- a/scripts/ci/jobs/tests-large.yml
+++ b/scripts/ci/jobs/tests-large.yml
@@ -4,17 +4,14 @@ tests large:
- docker
- envinf1
extends:
- - .defaults
- .rules-master-manual
-
+ dependencies: [meta_container]
variables:
BUILD_DIR: "build-tests-large"
CCACHE_DIR: "$CI_PROJECT_DIR/.ccache"
CMAKE_ARGS: "-DOGS_USE_CONAN=OFF"
-
image:
name: $CONTAINER_GCC_IMAGE
-
script:
- git lfs install
- git lfs fetch
@@ -27,11 +24,9 @@ tests large:
-DCMAKE_BUILD_TYPE=$BUILD_TYPE
-DOGS_BUILD_PROCESSES=$BUILD_PROCESSES
- cmake --build . --target ctest-large
-
artifacts:
paths:
- $BUILD_DIR/Tests/ctest.xml
-
cache:
paths:
- $CCACHE_DIR
diff --git a/scripts/ci/jobs/web.yml b/scripts/ci/jobs/web.yml
index afa5d414aca..5fc59561ee9 100644
--- a/scripts/ci/jobs/web.yml
+++ b/scripts/ci/jobs/web.yml
@@ -5,7 +5,7 @@ web url checker:
- docker
extends:
- .rules-manual
- needs: []
+ dependencies: [meta_container]
image:
name: $WEB_IMAGE
script:
@@ -19,8 +19,7 @@ web url checker:
preview web site:
stage: build
image: $WEB_IMAGE
- extends:
- - .defaults
+ dependencies: [meta_container]
script:
# HACK to easier linking to the generated pages
- echo '<meta http-equiv="REFRESH" content="0;URL=web/public/index.html">' >> WebPreview.html
@@ -45,7 +44,7 @@ preview web site:
deploy web site:
stage: package
image: $WEB_IMAGE
- needs: []
+ dependencies: [meta_container]
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
changes:
--
GitLab