updating gitlab ci

reduced size of artifacts

moved deployment of pages to deploy stage

added test stage

split builds into separate jobs that run in parallel

updated dind images

separated into multiple build jobs / each build job per service

added script for running build jobs

closes #135 (closed) #134 (closed)

.gitlab-ci.yml

 variables:
   DOCKER_TLS_CERTDIR: ""
+  IMAGE_1: "$CI_REGISTRY_IMAGE/pvs_core"
+  IMAGE_2: "$CI_REGISTRY_IMAGE/pvs_preprocessor"
+  IMAGE_3: "$CI_REGISTRY_IMAGE/pvs_client"
+  IMAGE_4: "$CI_REGISTRY_IMAGE/pvs_cache"
+  IMAGE_5: "$CI_REGISTRY_IMAGE/fluentd"
+  IMAGE_6: "$CI_REGISTRY_IMAGE/pvs_ingestor"
+  IMAGE_7: "$CI_REGISTRY_IMAGE/pvs_sftp"
+  IMAGE_8: "$CI_REGISTRY_IMAGE/pvs_shibauth"
 
 stages:
   - build
+  - test
   - chart
   - deploy
 
-build-tag:
-  image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
+image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
+
+before_script:
+  - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+
+services:
+  - docker:20.10.8-dind
+
+build-core:
   stage: build
-  services:
-    - docker:19.03.13-dind
-  before_script:
-    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
-    - IMAGE_1="$CI_REGISTRY_IMAGE/pvs_core"
-    - docker pull "$IMAGE_1":latest || true
-    - docker build --cache-from "$IMAGE_1":latest -t "$IMAGE_1":dev -t "$IMAGE_1":$CI_COMMIT_TAG core/
-    - IMAGE_2="$CI_REGISTRY_IMAGE/pvs_preprocessor"
-    - docker pull "$IMAGE_2":latest || true
-    - docker build --cache-from "$IMAGE_2":latest -t "$IMAGE_2":dev -t "$IMAGE_2":$CI_COMMIT_TAG preprocessor/
-    - IMAGE_3="$CI_REGISTRY_IMAGE/pvs_client"
-    - docker pull "$IMAGE_3":latest || true
-    - docker build --cache-from "$IMAGE_3":latest -t "$IMAGE_3":dev -t "$IMAGE_3":$CI_COMMIT_TAG client/
-    - IMAGE_4="$CI_REGISTRY_IMAGE/pvs_cache"
-    - docker pull "$IMAGE_4":latest || true
-    - docker build --cache-from "$IMAGE_4":latest -t "$IMAGE_4":dev -t "$IMAGE_4":$CI_COMMIT_TAG cache/
-    - IMAGE_5="$CI_REGISTRY_IMAGE/fluentd"
-    - docker pull "$IMAGE_5":latest || true
-    - docker build --cache-from "$IMAGE_5":latest -t "$IMAGE_5":dev -t "$IMAGE_5":$CI_COMMIT_TAG fluentd/
-    - IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor"
-    - docker pull "$IMAGE_6":latest || true
-    - docker build --cache-from "$IMAGE_6":latest -t "$IMAGE_6":dev -t "$IMAGE_6":$CI_COMMIT_TAG ingestor/
-    - IMAGE_7="$CI_REGISTRY_IMAGE/pvs_sftp"
-    - docker pull "$IMAGE_7":latest || true
-    - docker build --cache-from "$IMAGE_7":latest -t "$IMAGE_7":dev -t "$IMAGE_7":$CI_COMMIT_TAG sftp/
-    - IMAGE_8="$CI_REGISTRY_IMAGE/pvs_shibauth"
-    - docker pull "$IMAGE_8":latest || true
-    - docker build --cache-from "$IMAGE_8":latest -t "$IMAGE_8":dev -t "$IMAGE_8":$CI_COMMIT_TAG shibauth/
-    - docker push "$IMAGE_1":$CI_COMMIT_TAG
-    - docker push "$IMAGE_2":$CI_COMMIT_TAG
-    - docker push "$IMAGE_3":$CI_COMMIT_TAG
-    - docker push "$IMAGE_4":$CI_COMMIT_TAG
-    - docker push "$IMAGE_5":$CI_COMMIT_TAG
-    - docker push "$IMAGE_6":$CI_COMMIT_TAG
-    - docker push "$IMAGE_7":$CI_COMMIT_TAG
-    - docker push "$IMAGE_8":$CI_COMMIT_TAG
-  only:
-    - tags
-build-master-staging:
-  image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_1 core
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-preprocessor:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_2 preprocessor
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-client:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_3 client
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-cache:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_4 cache
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-fluentd:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_5 fluentd
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-ingestor:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_6 ingestor
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-sftp:
   stage: build
-  services:
-    - docker:19.03.13-dind
-  before_script:
-    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
-    - if [[ "$CI_COMMIT_BRANCH" = "master" ]] ; then TAG_USED="latest"; else TAG_USED="staging"; fi
-    - IMAGE_1="$CI_REGISTRY_IMAGE/pvs_core"
-    - docker pull "$IMAGE_1":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_1":"$TAG_USED" -t "$IMAGE_1":dev -t "$IMAGE_1":"$TAG_USED" core/
-    - IMAGE_2="$CI_REGISTRY_IMAGE/pvs_preprocessor"
-    - docker pull "$IMAGE_2":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_2":"$TAG_USED" -t "$IMAGE_2":dev -t "$IMAGE_2":"$TAG_USED" preprocessor/
-    - IMAGE_3="$CI_REGISTRY_IMAGE/pvs_client"
-    - docker pull "$IMAGE_3":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_3":"$TAG_USED" -t "$IMAGE_3":dev -t "$IMAGE_3":"$TAG_USED" client/
-    - IMAGE_4="$CI_REGISTRY_IMAGE/pvs_cache"
-    - docker pull "$IMAGE_4":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_4":"$TAG_USED" -t "$IMAGE_4":dev -t "$IMAGE_4":"$TAG_USED" cache/
-    - IMAGE_5="$CI_REGISTRY_IMAGE/fluentd"
-    - docker pull "$IMAGE_5":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_5":"$TAG_USED" -t "$IMAGE_5":dev -t "$IMAGE_5":"$TAG_USED" fluentd/
-    - IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor"
-    - docker pull "$IMAGE_6":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_6":"$TAG_USED" -t "$IMAGE_6":dev -t "$IMAGE_6":"$TAG_USED" ingestor/
-    - IMAGE_7="$CI_REGISTRY_IMAGE/pvs_sftp"
-    - docker pull "$IMAGE_7":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_7":latest -t "$IMAGE_7":dev -t "$IMAGE_7":"$TAG_USED" sftp/
-    - IMAGE_8="$CI_REGISTRY_IMAGE/pvs_shibauth"
-    - docker pull "$IMAGE_8":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_8":latest -t "$IMAGE_8":dev -t "$IMAGE_8":"$TAG_USED" shibauth/
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_7 sftp
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-shibauth:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_8 shibauth
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+
+# master-staging - test in master and staging
+ms-test:
+  stage: test
+  script:
+    - docker pull $IMAGE_1
+    - docker pull $IMAGE_2
+    - docker pull $IMAGE_3
+    - docker pull $IMAGE_4
+    - docker pull $IMAGE_5
+    - docker pull $IMAGE_6
+    - docker pull $IMAGE_7
+    - docker pull $IMAGE_8
+    - docker tag $IMAGE_1 $IMAGE_1:dev
+    - docker tag $IMAGE_2 $IMAGE_2:dev
+    - docker tag $IMAGE_3 $IMAGE_3:dev
+    - docker tag $IMAGE_4 $IMAGE_4:dev
+    - docker tag $IMAGE_5 $IMAGE_5:dev
+    - docker tag $IMAGE_6 $IMAGE_6:dev
+    - docker tag $IMAGE_7 $IMAGE_7:dev
+    - docker tag $IMAGE_8 $IMAGE_8:dev
     - cd ./testing && ./gitlab_test.sh
     - if [ $? -ne 0 ]; then exit 1; fi  # actually fail build
-    - docker push "$IMAGE_1":"$TAG_USED"
-    - docker push "$IMAGE_2":"$TAG_USED"
-    - docker push "$IMAGE_3":"$TAG_USED"
-    - docker push "$IMAGE_4":"$TAG_USED"
-    - docker push "$IMAGE_5":"$TAG_USED"
-    - docker push "$IMAGE_6":"$TAG_USED"
-    - docker push "$IMAGE_7":"$TAG_USED"
-    - docker push "$IMAGE_8":"$TAG_USED"
   only:
     - staging
     - master
-  except:
-    - tags
   artifacts:
     when: always
     expire_in: 4 week
@@ -106,38 +123,13 @@ build-master-staging:
       - emg-pvs_preprocessor.txt
       - emg-pvs_registrar.txt
       - emg-pvs_renderer.txt
-build:
-  image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
-  stage: build
-  services:
-    - docker:19.03.13-dind
-  before_script:
-    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+
+# dev - test in branches only
+dev-test:
+  stage: test
   script:
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_core"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev core/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_preprocessor"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev preprocessor/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_client"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev client/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_cache"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev cache/
-    - IMAGE="$CI_REGISTRY_IMAGE/fluentd"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev fluentd/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_ingestor"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev ingestor/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_sftp"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev sftp/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_shibauth"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev shibauth/
+    - gunzip image/*
+    - for f in image/*.tar; do cat $f | docker load; done
     - cd ./testing && ./gitlab_test.sh
     - if [ $? -ne 0 ]; then exit 1; fi  # actually fail build
   except:
@@ -158,9 +150,12 @@ build:
       - emg-pvs_registrar.txt
       - emg-pvs_renderer.txt
 
-review-docs:
+
+# Pages deployment
+pages-review:
   image: python:3.8-slim
-  stage: build
+  stage: deploy
+  services: []
   script:
     - apt-get update && apt-get install -y build-essential rsync grsync git texlive-base texlive-latex-extra latexmk
     - git fetch origin
@@ -179,7 +174,7 @@ review-docs:
   environment:
     name: review/$CI_COMMIT_REF_NAME
     url: "https://$CI_PROJECT_ROOT_NAMESPACE.pages.eox.at/-/prism/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/$CI_DEFAULT_BRANCH/index.html"
-    on_stop: stop_review_app
+    on_stop: stop-pages-review
     auto_stop_in: 1 week
   variables:
     PUBLIC_URL: "/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public"
@@ -191,9 +186,10 @@ review-docs:
     - staging
   when: manual
 
-stop_review_app:
+stop-pages-review:
   stage: deploy
   script: echo "stopping review"
+  services: []
   environment:
     name: review/$CI_COMMIT_REF_NAME
     action: stop
@@ -207,6 +203,7 @@ stop_review_app:
 pages:
   image: python:3.8-slim
   stage: build
+  services: []
   script:
     - apt-get update && apt-get install -y build-essential rsync grsync git texlive-base texlive-latex-extra latexmk
     - git fetch origin
@@ -229,8 +226,10 @@ pages:
     - tags
     - staging
 
+# Helm chart publishing
 publish-helm-chart:
   image: devth/helm:v3.5.3
+  services: []
   stage: chart
   script:
     # push Helm chart