Merge branch 'ci_parallel_test' into 'staging'

updating gitlab ci

Closes #135 and #134

See merge request !92

.gitlab-ci.yml

 variables:
   DOCKER_TLS_CERTDIR: ""
+  IMAGE_1: "$CI_REGISTRY_IMAGE/pvs_core"
+  IMAGE_2: "$CI_REGISTRY_IMAGE/pvs_preprocessor"
+  IMAGE_3: "$CI_REGISTRY_IMAGE/pvs_client"
+  IMAGE_4: "$CI_REGISTRY_IMAGE/pvs_cache"
+  IMAGE_5: "$CI_REGISTRY_IMAGE/fluentd"
+  IMAGE_6: "$CI_REGISTRY_IMAGE/pvs_ingestor"
+  IMAGE_7: "$CI_REGISTRY_IMAGE/pvs_sftp"
+  IMAGE_8: "$CI_REGISTRY_IMAGE/pvs_shibauth"
 
 stages:
   - build
+  - test
   - chart
   - deploy
 
-build-tag:
-  image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
+image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
+
+before_script:
+  - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+
+services:
+  - docker:20.10.8-dind
+
+build-core:
   stage: build
-  services:
-    - docker:19.03.13-dind
-  before_script:
-    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
-    - IMAGE_1="$CI_REGISTRY_IMAGE/pvs_core"
-    - docker pull "$IMAGE_1":latest || true
-    - docker build --cache-from "$IMAGE_1":latest -t "$IMAGE_1":dev -t "$IMAGE_1":$CI_COMMIT_TAG core/
-    - IMAGE_2="$CI_REGISTRY_IMAGE/pvs_preprocessor"
-    - docker pull "$IMAGE_2":latest || true
-    - docker build --cache-from "$IMAGE_2":latest -t "$IMAGE_2":dev -t "$IMAGE_2":$CI_COMMIT_TAG preprocessor/
-    - IMAGE_3="$CI_REGISTRY_IMAGE/pvs_client"
-    - docker pull "$IMAGE_3":latest || true
-    - docker build --cache-from "$IMAGE_3":latest -t "$IMAGE_3":dev -t "$IMAGE_3":$CI_COMMIT_TAG client/
-    - IMAGE_4="$CI_REGISTRY_IMAGE/pvs_cache"
-    - docker pull "$IMAGE_4":latest || true
-    - docker build --cache-from "$IMAGE_4":latest -t "$IMAGE_4":dev -t "$IMAGE_4":$CI_COMMIT_TAG cache/
-    - IMAGE_5="$CI_REGISTRY_IMAGE/fluentd"
-    - docker pull "$IMAGE_5":latest || true
-    - docker build --cache-from "$IMAGE_5":latest -t "$IMAGE_5":dev -t "$IMAGE_5":$CI_COMMIT_TAG fluentd/
-    - IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor"
-    - docker pull "$IMAGE_6":latest || true
-    - docker build --cache-from "$IMAGE_6":latest -t "$IMAGE_6":dev -t "$IMAGE_6":$CI_COMMIT_TAG ingestor/
-    - IMAGE_7="$CI_REGISTRY_IMAGE/pvs_sftp"
-    - docker pull "$IMAGE_7":latest || true
-    - docker build --cache-from "$IMAGE_7":latest -t "$IMAGE_7":dev -t "$IMAGE_7":$CI_COMMIT_TAG sftp/
-    - IMAGE_8="$CI_REGISTRY_IMAGE/pvs_shibauth"
-    - docker pull "$IMAGE_8":latest || true
-    - docker build --cache-from "$IMAGE_8":latest -t "$IMAGE_8":dev -t "$IMAGE_8":$CI_COMMIT_TAG shibauth/
-    - docker push "$IMAGE_1":$CI_COMMIT_TAG
-    - docker push "$IMAGE_2":$CI_COMMIT_TAG
-    - docker push "$IMAGE_3":$CI_COMMIT_TAG
-    - docker push "$IMAGE_4":$CI_COMMIT_TAG
-    - docker push "$IMAGE_5":$CI_COMMIT_TAG
-    - docker push "$IMAGE_6":$CI_COMMIT_TAG
-    - docker push "$IMAGE_7":$CI_COMMIT_TAG
-    - docker push "$IMAGE_8":$CI_COMMIT_TAG
-  only:
-    - tags
-build-master-staging:
-  image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_1 core
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-preprocessor:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_2 preprocessor
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-client:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_3 client
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-cache:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_4 cache
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-fluentd:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_5 fluentd
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-ingestor:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_6 ingestor
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-sftp:
   stage: build
-  services:
-    - docker:19.03.13-dind
-  before_script:
-    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
   script:
-    - if [[ "$CI_COMMIT_BRANCH" = "master" ]] ; then TAG_USED="latest"; else TAG_USED="staging"; fi
-    - IMAGE_1="$CI_REGISTRY_IMAGE/pvs_core"
-    - docker pull "$IMAGE_1":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_1":"$TAG_USED" -t "$IMAGE_1":dev -t "$IMAGE_1":"$TAG_USED" core/
-    - IMAGE_2="$CI_REGISTRY_IMAGE/pvs_preprocessor"
-    - docker pull "$IMAGE_2":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_2":"$TAG_USED" -t "$IMAGE_2":dev -t "$IMAGE_2":"$TAG_USED" preprocessor/
-    - IMAGE_3="$CI_REGISTRY_IMAGE/pvs_client"
-    - docker pull "$IMAGE_3":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_3":"$TAG_USED" -t "$IMAGE_3":dev -t "$IMAGE_3":"$TAG_USED" client/
-    - IMAGE_4="$CI_REGISTRY_IMAGE/pvs_cache"
-    - docker pull "$IMAGE_4":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_4":"$TAG_USED" -t "$IMAGE_4":dev -t "$IMAGE_4":"$TAG_USED" cache/
-    - IMAGE_5="$CI_REGISTRY_IMAGE/fluentd"
-    - docker pull "$IMAGE_5":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_5":"$TAG_USED" -t "$IMAGE_5":dev -t "$IMAGE_5":"$TAG_USED" fluentd/
-    - IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor"
-    - docker pull "$IMAGE_6":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_6":"$TAG_USED" -t "$IMAGE_6":dev -t "$IMAGE_6":"$TAG_USED" ingestor/
-    - IMAGE_7="$CI_REGISTRY_IMAGE/pvs_sftp"
-    - docker pull "$IMAGE_7":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_7":latest -t "$IMAGE_7":dev -t "$IMAGE_7":"$TAG_USED" sftp/
-    - IMAGE_8="$CI_REGISTRY_IMAGE/pvs_shibauth"
-    - docker pull "$IMAGE_8":"$TAG_USED" || true
-    - docker build --cache-from "$IMAGE_8":latest -t "$IMAGE_8":dev -t "$IMAGE_8":"$TAG_USED" shibauth/
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_7 sftp
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+build-shibauth:
+  stage: build
+  script:
+    - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_8 shibauth
+  artifacts:
+    paths:
+      - "image"
+    expire_in: 5 hours
+
+# master-staging - test in master and staging
+ms-test:
+  stage: test
+  script:
+    - docker pull $IMAGE_1
+    - docker pull $IMAGE_2
+    - docker pull $IMAGE_3
+    - docker pull $IMAGE_4
+    - docker pull $IMAGE_5
+    - docker pull $IMAGE_6
+    - docker pull $IMAGE_7
+    - docker pull $IMAGE_8
+    - docker tag $IMAGE_1 $IMAGE_1:dev
+    - docker tag $IMAGE_2 $IMAGE_2:dev
+    - docker tag $IMAGE_3 $IMAGE_3:dev
+    - docker tag $IMAGE_4 $IMAGE_4:dev
+    - docker tag $IMAGE_5 $IMAGE_5:dev
+    - docker tag $IMAGE_6 $IMAGE_6:dev
+    - docker tag $IMAGE_7 $IMAGE_7:dev
+    - docker tag $IMAGE_8 $IMAGE_8:dev
     - cd ./testing && ./gitlab_test.sh
     - if [ $? -ne 0 ]; then exit 1; fi  # actually fail build
-    - docker push "$IMAGE_1":"$TAG_USED"
-    - docker push "$IMAGE_2":"$TAG_USED"
-    - docker push "$IMAGE_3":"$TAG_USED"
-    - docker push "$IMAGE_4":"$TAG_USED"
-    - docker push "$IMAGE_5":"$TAG_USED"
-    - docker push "$IMAGE_6":"$TAG_USED"
-    - docker push "$IMAGE_7":"$TAG_USED"
-    - docker push "$IMAGE_8":"$TAG_USED"
   only:
     - staging
     - master
-  except:
-    - tags
   artifacts:
     when: always
     expire_in: 4 week
@@ -106,38 +123,13 @@ build-master-staging:
       - emg-pvs_preprocessor.txt
       - emg-pvs_registrar.txt
       - emg-pvs_renderer.txt
-build:
-  image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
-  stage: build
-  services:
-    - docker:19.03.13-dind
-  before_script:
-    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+
+# dev - test in branches only
+dev-test:
+  stage: test
   script:
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_core"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev core/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_preprocessor"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev preprocessor/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_client"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev client/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_cache"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev cache/
-    - IMAGE="$CI_REGISTRY_IMAGE/fluentd"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev fluentd/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_ingestor"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev ingestor/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_sftp"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev sftp/
-    - IMAGE="$CI_REGISTRY_IMAGE/pvs_shibauth"
-    - docker pull "$IMAGE":staging || true
-    - docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev shibauth/
+    - gunzip image/*
+    - for f in image/*.tar; do cat $f | docker load; done
     - cd ./testing && ./gitlab_test.sh
     - if [ $? -ne 0 ]; then exit 1; fi  # actually fail build
   except:
@@ -158,9 +150,12 @@ build:
       - emg-pvs_registrar.txt
       - emg-pvs_renderer.txt
 
-review-docs:
+
+# Pages deployment
+pages-review:
   image: python:3.8-slim
-  stage: build
+  stage: deploy
+  services: []
   script:
     - apt-get update && apt-get install -y build-essential rsync grsync git texlive-base texlive-latex-extra latexmk
     - git fetch origin
@@ -179,7 +174,7 @@ review-docs:
   environment:
     name: review/$CI_COMMIT_REF_NAME
     url: "https://$CI_PROJECT_ROOT_NAMESPACE.pages.eox.at/-/prism/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/$CI_DEFAULT_BRANCH/index.html"
-    on_stop: stop_review_app
+    on_stop: stop-pages-review
     auto_stop_in: 1 week
   variables:
     PUBLIC_URL: "/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public"
@@ -191,9 +186,10 @@ review-docs:
     - staging
   when: manual
 
-stop_review_app:
+stop-pages-review:
   stage: deploy
   script: echo "stopping review"
+  services: []
   environment:
     name: review/$CI_COMMIT_REF_NAME
     action: stop
@@ -207,6 +203,7 @@ stop_review_app:
 pages:
   image: python:3.8-slim
   stage: build
+  services: []
   script:
     - apt-get update && apt-get install -y build-essential rsync grsync git texlive-base texlive-latex-extra latexmk
     - git fetch origin
@@ -229,8 +226,10 @@ pages:
     - tags
     - staging
 
+# Helm chart publishing
 publish-helm-chart:
   image: devth/helm:v3.5.3
+  services: []
   stage: chart
   script:
     # push Helm chart

ci_image_build.sh

+#!/bin/bash    
+
+CI_COMMIT_REF_NAME=$1
+IMAGE=$2
+SERVICE_SHORT_NAME=$3
+
+echo "service: $SERVICE_SHORT_NAME"
+echo "commit_ref: $CI_COMMIT_REF_NAME"
+echo "image: $IMAGE"
+
+TAG_REGEX="^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
+
+# tags
+if [[ $CI_COMMIT_REF_NAME =~ "$TAG_REGEX" ]];
+then
+    echo "Detected tag"
+    docker pull $IMAGE:latest || true
+    docker build --cache-from $IMAGE:latest -t "$IMAGE":dev -t $IMAGE:$CI_COMMIT_REF_NAME $SERVICE_SHORT_NAME/
+    docker push $IMAGE:$CI_COMMIT_REF_NAME
+
+# master/staging
+elif [[ $CI_COMMIT_REF_NAME = 'master' ]] || [[ $CI_COMMIT_REF_NAME = 'staging' ]];
+then
+    echo "Detected master/staging"
+    if [[ "$CI_COMMIT_REF_NAME" = "master" ]] ; then TAG_USED="latest"; else TAG_USED="staging"; fi
+    docker pull "$IMAGE":"$TAG_USED" || true
+    docker build --cache-from "$IMAGE":"$TAG_USED" -t "$IMAGE":dev -t "$IMAGE":"$TAG_USED" $SERVICE_SHORT_NAME/
+    docker push "$IMAGE":"$TAG_USED"
+
+# branches
+else
+    echo "Detected branch"
+    docker pull "$IMAGE":staging || true
+    docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev $SERVICE_SHORT_NAME/
+    mkdir image
+    docker save "$IMAGE":dev > image/$SERVICE_SHORT_NAME.tar
+    gzip image/$SERVICE_SHORT_NAME.tar
+fi

testing/Dockerfile

@@ -26,15 +26,17 @@
 #-----------------------------------------------------------------------------
 
 # this image is built manually and pushed to registry.gitlab.eox.at/esa/prism/vs/docker-base-testing
+FROM docker:20.10.8-dind
+
 LABEL name="prism view server docker base image" \
       vendor="EOX IT Services GmbH <https://eox.at>" \
       license="MIT Copyright (C) 2020 EOX IT Services GmbH <https://eox.at>" \
       type="prism view server docker base image" \
       version="1.4.8"
 
-FROM docker:19.03.13-dind
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
 
-RUN apk update && apk add bash postgresql-dev gcc g++ python3 python3-dev musl-dev py-pip libffi-dev openssl-dev make gdal==3.1.4-r0 gdal-dev==3.1.4-r0
+RUN apk update && apk add bash postgresql-dev gcc g++ python3 python3-dev musl-dev py-pip libffi-dev openssl-dev make gdal gdal-dev
 
 ADD requirements.txt /
 RUN pip3 install -r requirements.txt