EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 7ef91b35 authored by Nikola Jankovic's avatar Nikola Jankovic :computer:
Browse files

Merge branch 'ci_parallel_test' into 'staging' 

updating gitlab ci

Closes #135 and #134

See merge request !92
parents 8decd24b 2e438343
No related branches found
No related tags found
1 merge request!92updating gitlab ci
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 115
− 116
View file @ 7ef91b35
variables: variables:
DOCKER_TLS_CERTDIR: "" DOCKER_TLS_CERTDIR: ""
IMAGE_1: "$CI_REGISTRY_IMAGE/pvs_core"
IMAGE_2: "$CI_REGISTRY_IMAGE/pvs_preprocessor"
IMAGE_3: "$CI_REGISTRY_IMAGE/pvs_client"
IMAGE_4: "$CI_REGISTRY_IMAGE/pvs_cache"
IMAGE_5: "$CI_REGISTRY_IMAGE/fluentd"
IMAGE_6: "$CI_REGISTRY_IMAGE/pvs_ingestor"
IMAGE_7: "$CI_REGISTRY_IMAGE/pvs_sftp"
IMAGE_8: "$CI_REGISTRY_IMAGE/pvs_shibauth"
stages: stages:
- build - build
- test
- chart - chart
- deploy - deploy
build-tag: image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
services:
- docker:20.10.8-dind
build-core:
stage: build stage: build
services:
- docker:19.03.13-dind
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
script: script:
- IMAGE_1="$CI_REGISTRY_IMAGE/pvs_core" - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_1 core
- docker pull "$IMAGE_1":latest || true artifacts:
- docker build --cache-from "$IMAGE_1":latest -t "$IMAGE_1":dev -t "$IMAGE_1":$CI_COMMIT_TAG core/ paths:
- IMAGE_2="$CI_REGISTRY_IMAGE/pvs_preprocessor" - "image"
- docker pull "$IMAGE_2":latest || true expire_in: 5 hours
- docker build --cache-from "$IMAGE_2":latest -t "$IMAGE_2":dev -t "$IMAGE_2":$CI_COMMIT_TAG preprocessor/ build-preprocessor:
- IMAGE_3="$CI_REGISTRY_IMAGE/pvs_client" stage: build
- docker pull "$IMAGE_3":latest || true script:
- docker build --cache-from "$IMAGE_3":latest -t "$IMAGE_3":dev -t "$IMAGE_3":$CI_COMMIT_TAG client/ - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_2 preprocessor
- IMAGE_4="$CI_REGISTRY_IMAGE/pvs_cache" artifacts:
- docker pull "$IMAGE_4":latest || true paths:
- docker build --cache-from "$IMAGE_4":latest -t "$IMAGE_4":dev -t "$IMAGE_4":$CI_COMMIT_TAG cache/ - "image"
- IMAGE_5="$CI_REGISTRY_IMAGE/fluentd" expire_in: 5 hours
- docker pull "$IMAGE_5":latest || true build-client:
- docker build --cache-from "$IMAGE_5":latest -t "$IMAGE_5":dev -t "$IMAGE_5":$CI_COMMIT_TAG fluentd/ stage: build
- IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor" script:
- docker pull "$IMAGE_6":latest || true - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_3 client
- docker build --cache-from "$IMAGE_6":latest -t "$IMAGE_6":dev -t "$IMAGE_6":$CI_COMMIT_TAG ingestor/ artifacts:
- IMAGE_7="$CI_REGISTRY_IMAGE/pvs_sftp" paths:
- docker pull "$IMAGE_7":latest || true - "image"
- docker build --cache-from "$IMAGE_7":latest -t "$IMAGE_7":dev -t "$IMAGE_7":$CI_COMMIT_TAG sftp/ expire_in: 5 hours
- IMAGE_8="$CI_REGISTRY_IMAGE/pvs_shibauth" build-cache:
- docker pull "$IMAGE_8":latest || true stage: build
- docker build --cache-from "$IMAGE_8":latest -t "$IMAGE_8":dev -t "$IMAGE_8":$CI_COMMIT_TAG shibauth/ script:
- docker push "$IMAGE_1":$CI_COMMIT_TAG - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_4 cache
- docker push "$IMAGE_2":$CI_COMMIT_TAG artifacts:
- docker push "$IMAGE_3":$CI_COMMIT_TAG paths:
- docker push "$IMAGE_4":$CI_COMMIT_TAG - "image"
- docker push "$IMAGE_5":$CI_COMMIT_TAG expire_in: 5 hours
- docker push "$IMAGE_6":$CI_COMMIT_TAG build-fluentd:
- docker push "$IMAGE_7":$CI_COMMIT_TAG stage: build
- docker push "$IMAGE_8":$CI_COMMIT_TAG script:
only: - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_5 fluentd
- tags artifacts:
build-master-staging: paths:
image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest - "image"
expire_in: 5 hours
build-ingestor:
stage: build
script:
- ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_6 ingestor
artifacts:
paths:
- "image"
expire_in: 5 hours
build-sftp:
stage: build stage: build
services:
- docker:19.03.13-dind
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
script: script:
- if [[ "$CI_COMMIT_BRANCH" = "master" ]] ; then TAG_USED="latest"; else TAG_USED="staging"; fi - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_7 sftp
- IMAGE_1="$CI_REGISTRY_IMAGE/pvs_core" artifacts:
- docker pull "$IMAGE_1":"$TAG_USED" || true paths:
- docker build --cache-from "$IMAGE_1":"$TAG_USED" -t "$IMAGE_1":dev -t "$IMAGE_1":"$TAG_USED" core/ - "image"
- IMAGE_2="$CI_REGISTRY_IMAGE/pvs_preprocessor" expire_in: 5 hours
- docker pull "$IMAGE_2":"$TAG_USED" || true build-shibauth:
- docker build --cache-from "$IMAGE_2":"$TAG_USED" -t "$IMAGE_2":dev -t "$IMAGE_2":"$TAG_USED" preprocessor/ stage: build
- IMAGE_3="$CI_REGISTRY_IMAGE/pvs_client" script:
- docker pull "$IMAGE_3":"$TAG_USED" || true - ./ci_image_build.sh $CI_COMMIT_REF_NAME $IMAGE_8 shibauth
- docker build --cache-from "$IMAGE_3":"$TAG_USED" -t "$IMAGE_3":dev -t "$IMAGE_3":"$TAG_USED" client/ artifacts:
- IMAGE_4="$CI_REGISTRY_IMAGE/pvs_cache" paths:
- docker pull "$IMAGE_4":"$TAG_USED" || true - "image"
- docker build --cache-from "$IMAGE_4":"$TAG_USED" -t "$IMAGE_4":dev -t "$IMAGE_4":"$TAG_USED" cache/ expire_in: 5 hours
- IMAGE_5="$CI_REGISTRY_IMAGE/fluentd"
- docker pull "$IMAGE_5":"$TAG_USED" || true # master-staging - test in master and staging
- docker build --cache-from "$IMAGE_5":"$TAG_USED" -t "$IMAGE_5":dev -t "$IMAGE_5":"$TAG_USED" fluentd/ ms-test:
- IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor" stage: test
- docker pull "$IMAGE_6":"$TAG_USED" || true script:
- docker build --cache-from "$IMAGE_6":"$TAG_USED" -t "$IMAGE_6":dev -t "$IMAGE_6":"$TAG_USED" ingestor/ - docker pull $IMAGE_1
- IMAGE_7="$CI_REGISTRY_IMAGE/pvs_sftp" - docker pull $IMAGE_2
- docker pull "$IMAGE_7":"$TAG_USED" || true - docker pull $IMAGE_3
- docker build --cache-from "$IMAGE_7":latest -t "$IMAGE_7":dev -t "$IMAGE_7":"$TAG_USED" sftp/ - docker pull $IMAGE_4
- IMAGE_8="$CI_REGISTRY_IMAGE/pvs_shibauth" - docker pull $IMAGE_5
- docker pull "$IMAGE_8":"$TAG_USED" || true - docker pull $IMAGE_6
- docker build --cache-from "$IMAGE_8":latest -t "$IMAGE_8":dev -t "$IMAGE_8":"$TAG_USED" shibauth/ - docker pull $IMAGE_7
- docker pull $IMAGE_8
- docker tag $IMAGE_1 $IMAGE_1:dev
- docker tag $IMAGE_2 $IMAGE_2:dev
- docker tag $IMAGE_3 $IMAGE_3:dev
- docker tag $IMAGE_4 $IMAGE_4:dev
- docker tag $IMAGE_5 $IMAGE_5:dev
- docker tag $IMAGE_6 $IMAGE_6:dev
- docker tag $IMAGE_7 $IMAGE_7:dev
- docker tag $IMAGE_8 $IMAGE_8:dev
- cd ./testing && ./gitlab_test.sh - cd ./testing && ./gitlab_test.sh
- if [ $? -ne 0 ]; then exit 1; fi # actually fail build - if [ $? -ne 0 ]; then exit 1; fi # actually fail build
- docker push "$IMAGE_1":"$TAG_USED"
- docker push "$IMAGE_2":"$TAG_USED"
- docker push "$IMAGE_3":"$TAG_USED"
- docker push "$IMAGE_4":"$TAG_USED"
- docker push "$IMAGE_5":"$TAG_USED"
- docker push "$IMAGE_6":"$TAG_USED"
- docker push "$IMAGE_7":"$TAG_USED"
- docker push "$IMAGE_8":"$TAG_USED"
only: only:
- staging - staging
- master - master
except:
- tags
artifacts: artifacts:
when: always when: always
expire_in: 4 week expire_in: 4 week
...@@ -106,38 +123,13 @@ build-master-staging: ...@@ -106,38 +123,13 @@ build-master-staging:
- emg-pvs_preprocessor.txt - emg-pvs_preprocessor.txt
- emg-pvs_registrar.txt - emg-pvs_registrar.txt
- emg-pvs_renderer.txt - emg-pvs_renderer.txt
build:
image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest # dev - test in branches only
stage: build dev-test:
services: stage: test
- docker:19.03.13-dind
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
script: script:
- IMAGE="$CI_REGISTRY_IMAGE/pvs_core" - gunzip image/*
- docker pull "$IMAGE":staging || true - for f in image/*.tar; do cat $f | docker load; done
- docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev core/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_preprocessor"
- docker pull "$IMAGE":staging || true
- docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev preprocessor/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_client"
- docker pull "$IMAGE":staging || true
- docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev client/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_cache"
- docker pull "$IMAGE":staging || true
- docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev cache/
- IMAGE="$CI_REGISTRY_IMAGE/fluentd"
- docker pull "$IMAGE":staging || true
- docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev fluentd/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_ingestor"
- docker pull "$IMAGE":staging || true
- docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev ingestor/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_sftp"
- docker pull "$IMAGE":staging || true
- docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev sftp/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_shibauth"
- docker pull "$IMAGE":staging || true
- docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev shibauth/
- cd ./testing && ./gitlab_test.sh - cd ./testing && ./gitlab_test.sh
- if [ $? -ne 0 ]; then exit 1; fi # actually fail build - if [ $? -ne 0 ]; then exit 1; fi # actually fail build
except: except:
...@@ -158,9 +150,12 @@ build: ...@@ -158,9 +150,12 @@ build:
- emg-pvs_registrar.txt - emg-pvs_registrar.txt
- emg-pvs_renderer.txt - emg-pvs_renderer.txt
review-docs:
# Pages deployment
pages-review:
image: python:3.8-slim image: python:3.8-slim
stage: build stage: deploy
services: []
script: script:
- apt-get update && apt-get install -y build-essential rsync grsync git texlive-base texlive-latex-extra latexmk - apt-get update && apt-get install -y build-essential rsync grsync git texlive-base texlive-latex-extra latexmk
- git fetch origin - git fetch origin
...@@ -179,7 +174,7 @@ review-docs: ...@@ -179,7 +174,7 @@ review-docs:
environment: environment:
name: review/$CI_COMMIT_REF_NAME name: review/$CI_COMMIT_REF_NAME
url: "https://$CI_PROJECT_ROOT_NAMESPACE.pages.eox.at/-/prism/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/$CI_DEFAULT_BRANCH/index.html" url: "https://$CI_PROJECT_ROOT_NAMESPACE.pages.eox.at/-/prism/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public/$CI_DEFAULT_BRANCH/index.html"
on_stop: stop_review_app on_stop: stop-pages-review
auto_stop_in: 1 week auto_stop_in: 1 week
variables: variables:
PUBLIC_URL: "/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public" PUBLIC_URL: "/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/public"
...@@ -191,9 +186,10 @@ review-docs: ...@@ -191,9 +186,10 @@ review-docs:
- staging - staging
when: manual when: manual
stop_review_app: stop-pages-review:
stage: deploy stage: deploy
script: echo "stopping review" script: echo "stopping review"
services: []
environment: environment:
name: review/$CI_COMMIT_REF_NAME name: review/$CI_COMMIT_REF_NAME
action: stop action: stop
...@@ -207,6 +203,7 @@ stop_review_app: ...@@ -207,6 +203,7 @@ stop_review_app:
pages: pages:
image: python:3.8-slim image: python:3.8-slim
stage: build stage: build
services: []
script: script:
- apt-get update && apt-get install -y build-essential rsync grsync git texlive-base texlive-latex-extra latexmk - apt-get update && apt-get install -y build-essential rsync grsync git texlive-base texlive-latex-extra latexmk
- git fetch origin - git fetch origin
...@@ -229,8 +226,10 @@ pages: ...@@ -229,8 +226,10 @@ pages:
- tags - tags
- staging - staging
# Helm chart publishing
publish-helm-chart: publish-helm-chart:
image: devth/helm:v3.5.3 image: devth/helm:v3.5.3
services: []
stage: chart stage: chart
script: script:
# push Helm chart # push Helm chart
......
ci_image_build.sh 0 → 100755
+ 38
− 0
View file @ 7ef91b35
#!/bin/bash
CI_COMMIT_REF_NAME=$1
IMAGE=$2
SERVICE_SHORT_NAME=$3
echo "service: $SERVICE_SHORT_NAME"
echo "commit_ref: $CI_COMMIT_REF_NAME"
echo "image: $IMAGE"
TAG_REGEX="^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
# tags
if [[ $CI_COMMIT_REF_NAME =~ "$TAG_REGEX" ]];
then
echo "Detected tag"
docker pull $IMAGE:latest || true
docker build --cache-from $IMAGE:latest -t "$IMAGE":dev -t $IMAGE:$CI_COMMIT_REF_NAME $SERVICE_SHORT_NAME/
docker push $IMAGE:$CI_COMMIT_REF_NAME
# master/staging
elif [[ $CI_COMMIT_REF_NAME = 'master' ]] || [[ $CI_COMMIT_REF_NAME = 'staging' ]];
then
echo "Detected master/staging"
if [[ "$CI_COMMIT_REF_NAME" = "master" ]] ; then TAG_USED="latest"; else TAG_USED="staging"; fi
docker pull "$IMAGE":"$TAG_USED" || true
docker build --cache-from "$IMAGE":"$TAG_USED" -t "$IMAGE":dev -t "$IMAGE":"$TAG_USED" $SERVICE_SHORT_NAME/
docker push "$IMAGE":"$TAG_USED"
# branches
else
echo "Detected branch"
docker pull "$IMAGE":staging || true
docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev $SERVICE_SHORT_NAME/
mkdir image
docker save "$IMAGE":dev > image/$SERVICE_SHORT_NAME.tar
gzip image/$SERVICE_SHORT_NAME.tar
fi
testing/Dockerfile
+ 4
− 2
View file @ 7ef91b35
...@@ -26,15 +26,17 @@ ...@@ -26,15 +26,17 @@
#----------------------------------------------------------------------------- #-----------------------------------------------------------------------------
# this image is built manually and pushed to registry.gitlab.eox.at/esa/prism/vs/docker-base-testing # this image is built manually and pushed to registry.gitlab.eox.at/esa/prism/vs/docker-base-testing
FROM docker:20.10.8-dind
LABEL name="prism view server docker base image" \ LABEL name="prism view server docker base image" \
vendor="EOX IT Services GmbH <https://eox.at>" \ vendor="EOX IT Services GmbH <https://eox.at>" \
license="MIT Copyright (C) 2020 EOX IT Services GmbH <https://eox.at>" \ license="MIT Copyright (C) 2020 EOX IT Services GmbH <https://eox.at>" \
type="prism view server docker base image" \ type="prism view server docker base image" \
version="1.4.8" version="1.4.8"
FROM docker:19.03.13-dind ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
RUN apk update && apk add bash postgresql-dev gcc g++ python3 python3-dev musl-dev py-pip libffi-dev openssl-dev make gdal==3.1.4-r0 gdal-dev==3.1.4-r0 RUN apk update && apk add bash postgresql-dev gcc g++ python3 python3-dev musl-dev py-pip libffi-dev openssl-dev make gdal gdal-dev
ADD requirements.txt / ADD requirements.txt /
RUN pip3 install -r requirements.txt RUN pip3 install -r requirements.txt
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment