[ci] Docker image generation for MRs.

ff695bd7 · Lars Bilke · 46e847f8 · ff695bd7 · ff695bd7 · ff695bd7
Verified Commit ff695bd7 authored 4 years ago by Lars Bilke
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
 stages:
-  - pre-check
+  - preparation
  - build
  - check
  - gui
@@ -11,7 +11,6 @@ variables:
  CONTAINER_GCC_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/gcc
  CONTAINER_GCC_GUI_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/gcc-gui
  CONTAINER_CLANG_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/clang
-  CONTAINER_TAG: latest
  XUNIT_TO_JUNIT_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/xunit-to-junit
  PRECOMMIT_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/pre-commit
  WEB_IMAGE: $CI_REGISTRY/ogs/$CI_PROJECT_NAME/web

--- a/scripts/ci/extends/template-build-dockerfile.yml
+++ b/scripts/ci/extends/template-build-dockerfile.yml
@@ -15,8 +15,6 @@
      - scripts/ci/extends/template-build-dockerfile.yml
  tags:
    - docker-shell
-  variables:
-    GIT_LFS_SKIP_SMUDGE: "1"
  script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker pull $IMAGE_REF || true

--- a/scripts/ci/extends/template-build-linux.yml
+++ b/scripts/ci/extends/template-build-linux.yml
@@ -4,7 +4,7 @@
    - docker
  extends:
    - .test-artifacts
+  dependencies: [meta, meta_container]
  variables:
    CCACHE_DIR: "$CI_PROJECT_DIR/.ccache"

--- a/scripts/ci/extends/template-build-win.yml
+++ b/scripts/ci/extends/template-build-win.yml
@@ -4,6 +4,7 @@
    - windows
  extends:
    - .vs2019-environment
+  dependencies: [meta, meta_container]
  variables:
  script:
    - mkdir -p $env:BUILD_DIR

--- a/scripts/ci/jobs/build-docs.yml
+++ b/scripts/ci/jobs/build-docs.yml
@@ -2,18 +2,15 @@ build docs:
  stage: build
  image:
    name: $CONTAINER_GCC_IMAGE
-  needs: [meta]
+  dependencies: [meta, meta_container]
  before_script:
    # HACK to easier linking to the generated pages
    - echo '<meta http-equiv="REFRESH" content="0;URL=build/docs/index.html">' >> Doxygen.html
    - mkdir -p build
    - cd build
  script:
    - cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DOGS_USE_CONAN=OFF -DOGS_BUILD_PROCESSES=GroundwaterFlow
    - cmake --build . --target doc > >(tee make-docs.output)
  artifacts:
    expose_as: 'Doxygen'
    paths:
@@ -24,8 +21,7 @@ build docs:
 check docs links:
  stage: check
  image: $WEB_IMAGE
-  needs:
+  dependencies: [meta_container, "build docs"]
-    - job: build docs
  extends:
      - .rules-master-manual
  script:
@@ -42,8 +38,7 @@ deploy doxygen:
  image: $WEB_IMAGE
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
-  needs:
+  dependencies: [meta_container, "build docs"]
-    - job: build docs
  variables:
    GIT_STRATEGY: none
  script:

--- a/scripts/ci/jobs/build-gui-linux.yml
+++ b/scripts/ci/jobs/build-gui-linux.yml
@@ -6,10 +6,6 @@ build gui linux:
  tags:
    - envinf1
    - docker
-  needs:
-    - meta
-    - job: build linux
-      artifacts: false
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
    - changes:

--- a/scripts/ci/jobs/build-gui-mac.yml
+++ b/scripts/ci/jobs/build-gui-mac.yml
 build gui mac:
  extends: .template-build-linux
  stage: gui
-  needs:
-    - meta
-    - job: build mac
-      artifacts: false
  tags:
    - mac
  rules:

--- a/scripts/ci/jobs/build-gui-win.yml
+++ b/scripts/ci/jobs/build-gui-win.yml
 build gui win:
  extends: .template-build-win
  stage: gui
-  needs:
-    - meta
-    - job: build win
-      artifacts: false
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
    - changes:

--- a/scripts/ci/jobs/build-linux-frontend.yml
+++ b/scripts/ci/jobs/build-linux-frontend.yml
@@ -4,7 +4,6 @@ build linux frontend:
    - .rules-master-manual
  tags:
    - frontend2
-  needs: [meta]
  variables:
    BUILD_DIR: "build"
    CCACHE_DIR: "/data/ogs/.ccache"

--- a/scripts/ci/jobs/build-linux-petsc.yml
+++ b/scripts/ci/jobs/build-linux-petsc.yml
@@ -2,7 +2,6 @@ build linux petsc:
  extends: .template-build-linux
  tags:
    - envinf1-shell
-  needs: [meta]
  variables:
    BUILD_DIR: "build-petsc"
    CMAKE_ARGS: "-DOGS_USE_CONAN=OFF -DOGS_USE_PETSC=ON -DBUILD_SHARED_LIBS=ON"
--- a/scripts/ci/jobs/build-linux.yml
+++ b/scripts/ci/jobs/build-linux.yml
@@ -11,4 +11,3 @@ build linux:
      -DOGS_USE_PYTHON=ON
  image:
    name: $CONTAINER_GCC_IMAGE
-  needs: [meta]
--- a/scripts/ci/jobs/build-mac.yml
+++ b/scripts/ci/jobs/build-mac.yml
@@ -2,7 +2,6 @@ build mac:
  extends: .template-build-linux
  tags:
    - mac
-  needs: [meta]
  variables:
    BUILD_DIR: "build-mac"
    CMAKE_ARGS: >-

--- a/scripts/ci/jobs/build-win.yml
+++ b/scripts/ci/jobs/build-win.yml
 build win:
  extends: .template-build-win
-  needs: [meta]
  variables:
    BUILD_DIR: build-win
    CMAKE_ARGS: >-

--- a/scripts/ci/jobs/build_image.yml
+++ b/scripts/ci/jobs/build_image.yml
 build container images:
-  stage: .pre
+  stage: preparation
  rules:
-    # TODO
+    - changes:
-    # - changes:
+      - ThirdParty/container-maker
-      # - ThirdParty/container-maker
+      - scripts/ci/jobs/build_image.yml
-      # - scripts/ci/jobs/build_image.yml
+  dependencies: [meta_container]
-      # when: always
+  tags: [docker-shell]
-    - if: '$CI_COMMIT_BRANCH == "master"'
-      when: always
-  tags:
-    - envinf1-shell # can be changed back when instance-level vars are implemented
-  variables:
-    GIT_LFS_SKIP_SMUDGE: "1"
  extends: .container-maker-setup
  script:
-    - docker login -u $OGS_CI_REGISTRY_USER -p $OGS_CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker pull registry.opengeosys.org/ogs/ogs/ogs/gcc || true
    - python ThirdParty/container-maker/ogscm/cli.py --build --ogs off
-      --pm system --cvode --cppcheck --docs --gcovr
+      --pm system --cvode --cppcheck --docs --gcovr --tfel
-      --tag $CONTAINER_GCC_IMAGE:$CONTAINER_TAG --upload
+      --tag $CONTAINER_GCC_IMAGE --upload
-    - docker pull registry.opengeosys.org/ogs/ogs/ogs/gcc-gui || true
    - python ThirdParty/container-maker/ogscm/cli.py --build --ogs off
      --pm system --cvode --cppcheck --docs --gcovr --gui
-      --tag $CONTAINER_GCC_GUI_IMAGE:$CONTAINER_TAG --upload
+      --tag $CONTAINER_GCC_GUI_IMAGE --upload
-    - docker pull registry.opengeosys.org/ogs/ogs/ogs/clang || true
    - python ThirdParty/container-maker/ogscm/cli.py --build --ogs off
      --compiler clang --compiler_version 9
-      --tag $CONTAINER_CLANG_IMAGE:$CONTAINER_TAG --upload
+      --tag $CONTAINER_CLANG_IMAGE --upload
 ### global project images ###
 build xunit-to-junit image:
@@ -39,14 +29,10 @@ build xunit-to-junit image:
      - scripts/ci/jobs/build_image.yml
  tags:
    - docker-shell
-  variables:
-    GIT_LFS_SKIP_SMUDGE: "1"
  script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker pull $XUNIT_TO_JUNIT_IMAGE:latest || true
+    - docker build --cache-from $XUNIT_TO_JUNIT_IMAGE -t $XUNIT_TO_JUNIT_IMAGE -f scripts/docker/saxon/Dockerfile scripts/docker/saxon
-    - docker build --cache-from $XUNIT_TO_JUNIT_IMAGE:latest -t $XUNIT_TO_JUNIT_IMAGE:latest -f scripts/docker/saxon/Dockerfile scripts/docker/saxon
+    - docker push $XUNIT_TO_JUNIT_IMAGE
-    - docker push $XUNIT_TO_JUNIT_IMAGE:latest
 build pre-commit image:
@@ -58,13 +44,10 @@ build pre-commit image:
      - scripts/ci/jobs/build_image.yml
  tags:
    - docker-shell
-  variables:
-    GIT_LFS_SKIP_SMUDGE: "1"
  script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker pull $PRECOMMIT_IMAGE:latest || true
+    - docker build --cache-from $PRECOMMIT_IMAGE -t $PRECOMMIT_IMAGE -f scripts/docker/Dockerfile.pre-commit scripts/docker
-    - docker build --cache-from $PRECOMMIT_IMAGE:latest -t $PRECOMMIT_IMAGE:latest -f scripts/docker/Dockerfile.pre-commit scripts/docker
+    - docker push $PRECOMMIT_IMAGE
-    - docker push $PRECOMMIT_IMAGE:latest
 build web image:
  stage: .pre
@@ -75,10 +58,7 @@ build web image:
      - scripts/ci/jobs/build_image.yml
  tags:
    - docker-shell
-  variables:
-    GIT_LFS_SKIP_SMUDGE: "1"
  script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker pull $WEB_IMAGE:latest || true
+    - docker build --cache-from $WEB_IMAGE -t $WEB_IMAGE -f scripts/docker/Dockerfile.web scripts/docker
-    - docker build --cache-from $WEB_IMAGE:latest -t $WEB_IMAGE:latest -f scripts/docker/Dockerfile.web scripts/docker
+    - docker push $WEB_IMAGE
-    - docker push $WEB_IMAGE:latest
--- a/scripts/ci/jobs/check-header.yml
+++ b/scripts/ci/jobs/check-header.yml
@@ -4,17 +4,14 @@ check header:
  tags:
    - docker
  extends:
-    - .defaults
    - .rules-master-manual
+  dependencies: [meta_container]
  variables:
    BUILD_DIR: "build-check-header"
    CCACHE_DIR: "$CI_PROJECT_DIR/.ccache"
    CMAKE_ARGS: "-DOGS_CHECK_HEADER_COMPILATION=ON -DOGS_BUILD_UTILS=ON -DOGS_BUILD_GUI=ON -DOGS_USE_PYTHON=ON -DBUILD_SHARED_LIBS=ON"
  image:
    name: $CONTAINER_GCC_GUI_IMAGE
  script:
    - mkdir -p $BUILD_DIR
    - cd $BUILD_DIR
@@ -24,7 +21,6 @@ check header:
      -DCMAKE_BUILD_TYPE=$BUILD_TYPE
      -DOGS_BUILD_PROCESSES=$BUILD_PROCESSES || true
    - cat CMakeFiles/CMakeError.log
  cache:
    paths:
      - $CCACHE_DIR
--- a/scripts/ci/jobs/checks.yml
+++ b/scripts/ci/jobs/checks.yml
@@ -17,7 +17,6 @@ compiler warnings:
    - "if [[ $(cat make.output | grep warning -i) ]]; then echo 'There were Clang (macOS) compiler warnings:'; cat make.output | grep warning -i; exit_code=1; fi"
    - exit $exit_code
 ctest results:
  stage: check
  variables:

--- a/scripts/ci/jobs/clang-sanitizer.yml
+++ b/scripts/ci/jobs/clang-sanitizer.yml
 clang sanitizer:
  stage: check
-  tags:
-    - docker
  extends:
-    - .defaults
    - .rules-manual
+  dependencies: [meta_container]
  variables:
    BUILD_DIR: "build-sanitizer"
    CMAKE_ARGS: "-DOGS_ADDRESS_SANITIZER=ON -DOGS_UNDEFINED_BEHAVIOR_SANITIZER=ON -DOGS_BUILD_UTILS=ON"
    UBSAN_OPTIONS: "print_stacktrace=1"
    LSAN_OPTIONS: "suppressions=$CI_PROJECT_DIR/scripts/test/leak_sanitizer.suppressions"
  image:
    name: $CONTAINER_CLANG_IMAGE
  script:
    - mkdir -p $BUILD_DIR
    - cd $BUILD_DIR

--- a/scripts/ci/jobs/clang-tidy.yml
+++ b/scripts/ci/jobs/clang-tidy.yml
 clang tidy:
  stage: check
-  tags:
-    - docker
  extends:
-    - .defaults
    - .rules-manual
+  dependencies: [meta_container]
  variables:
    BUILD_DIR: "build-tidy"
    CMAKE_ARGS: "-DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF -DCMAKE_CXX_CLANG_TIDY=clang-tidy-9"

--- a/scripts/ci/jobs/code-quality.yml
+++ b/scripts/ci/jobs/code-quality.yml
 code_quality:
  stage: check
-  extends:
+  needs: []
-    - .defaults
  rules:
  variables:
    DOCKER_TLS_CERTDIR: "/certs"
--- a/scripts/ci/jobs/container.yml
+++ b/scripts/ci/jobs/container.yml
@@ -9,7 +9,6 @@ container:
  variables:
    OPENMPI_VERSIONS: 'off' # 4.0.1
  extends:
-    - .defaults
    - .container-maker-setup
  script:
    - python ThirdParty/container-maker/ogscm/cli.py -B -C -R --ogs . --pm system --cvode --ompi $OPENMPI_VERSIONS