From 6d0dd292eff95021a3d19ff3c44b96634ddeda5a Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Thu, 5 Nov 2020 10:10:42 +0100 Subject: [PATCH 1/3] use :dev tag for dev stack --- README.md | 35 +++++++++++++++++------------------ docker-compose.dem.dev.yml | 9 +++++++++ docker-compose.emg.dev.yml | 9 +++++++++ docker-compose.vhr18.dev.yml | 9 +++++++++ 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 28711c06..fff3c496 100644 --- a/README.md +++ b/README.md @@ -187,25 +187,17 @@ docker swarm init # initialize swarm ``` Build images: +Note we use **dev** tag for local development, so images need to be built locally ``` -docker build core/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_core -t registry.gitlab.eox.at/esa/prism/vs/pvs_core -docker build cache/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_cache -t registry.gitlab.eox.at/esa/prism/vs/pvs_cache -docker build preprocessor/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor -t registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor -docker build client/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_client -t registry.gitlab.eox.at/esa/prism/vs/pvs_client -docker build fluentd/ --cache-from registry.gitlab.eox.at/esa/prism/vs/fluentd -t registry.gitlab.eox.at/esa/prism/vs/fluentd -docker build ingestor/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor -t registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor - -``` -Or pull them from the registry: -``` -docker login -u {DOCKER_USER} -p {DOCKER_PASSWORD} registry.gitlab.eox.at -docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_core -docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_cache -docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor -docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_client -docker pull registry.gitlab.eox.at/esa/prism/vs/fluentd -docker pull registry.gitlab.eox.at/esa/prism/vs/ingestor +docker build core/ -t registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev +docker build cache/ -t registry.gitlab.eox.at/esa/prism/vs/pvs_cache:dev +docker build preprocessor/ -t registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:dev +docker build client/ -t registry.gitlab.eox.at/esa/prism/vs/pvs_client:dev +docker build fluentd/ -t registry.gitlab.eox.at/esa/prism/vs/fluentd:dev +docker build ingestor/ -t registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev ``` +For production deployment, as registry is open to public, this part is done by a later step `Deploy the stack in production` as it will pull necessary images automatically. + Create external network for stack to run: ``` docker network create -d overlay vhr18-extnet @@ -239,10 +231,17 @@ Deploy the stack in dev environment: docker stack deploy -c docker-compose.vhr18.yml -c docker-compose.vhr18.dev.yml -c docker-compose.logging.yml -c docker-compose.logging.dev.yml vhr18-pvs # start VHR_IMAGE_2018 stack in dev mode, for example to use local sources docker stack deploy -c docker-compose.emg.yml -c docker-compose.emg.dev.yml -c docker-compose.logging.yml -c docker-compose.logging.dev.yml emg-pvs # start Emergency stack in dev mode, for example to use local sources ``` -Deploy base stack in production environment: +Deploy base & logging stack in production environment: ``` docker stack deploy -c docker-compose.base.ops.yml base-pvs +docker stack deploy -c docker-compose.logging.yml docker-compose.logging.ops.yml logging ``` +Deploy the stack in production environment: +Please note that in order to reuse existing database volumes, needs to be the same. Here we use `vhr18-pvs` but in operational service `vhr18-pdas` is used. +``` +docker stack deploy -c docker-compose.vhr18.yml -c docker-compose.vhr18.ops.yml vhr18-pvs +``` + First steps: ``` # To register first data, use the following command inside the registrar container: diff --git a/docker-compose.dem.dev.yml b/docker-compose.dem.dev.yml index 8ac49a6c..56ee0072 100644 --- a/docker-compose.dem.dev.yml +++ b/docker-compose.dem.dev.yml @@ -9,7 +9,12 @@ services: - type: bind source: ./data/ target: /data/ + ingestor: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev + fluentd: + image: registry.gitlab.eox.at/esa/prism/vs/fluentd:dev client: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:dev ports: - "80:80" configs: @@ -20,6 +25,7 @@ services: source: ./data/ target: /data/ renderer: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev ports: - "81:80" - "82:8080" @@ -28,6 +34,7 @@ services: source: ./data/ target: /data/ registrar: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev volumes: - type: bind source: ./data/ @@ -38,6 +45,7 @@ services: logging: driver: "fluentd" cache: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_cache:dev ports: - "83:80" volumes: @@ -48,6 +56,7 @@ services: - source: mapcache-dev target: /mapcache-template.xml preprocessor: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:dev volumes: - type: tmpfs target: /tmp diff --git a/docker-compose.emg.dev.yml b/docker-compose.emg.dev.yml index 08b23bf0..672a23d5 100644 --- a/docker-compose.emg.dev.yml +++ b/docker-compose.emg.dev.yml @@ -9,7 +9,12 @@ services: - type: bind source: ./data/ target: /data/ + ingestor: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev + fluentd: + image: registry.gitlab.eox.at/esa/prism/vs/fluentd:dev client: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:dev ports: - "80:80" configs: @@ -20,6 +25,7 @@ services: source: ./data/ target: /data/ renderer: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev ports: - "81:80" - "82:8080" @@ -28,6 +34,7 @@ services: source: ./data/ target: /data/ registrar: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev volumes: - type: bind source: ./data/ @@ -36,6 +43,7 @@ services: source: ./core/ target: /core/ cache: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_cache:dev ports: - "83:80" volumes: @@ -46,6 +54,7 @@ services: - source: mapcache-dev target: /mapcache-template.xml preprocessor: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:dev volumes: - type: tmpfs target: /tmp diff --git a/docker-compose.vhr18.dev.yml b/docker-compose.vhr18.dev.yml index e7c46c3f..e37d1238 100644 --- a/docker-compose.vhr18.dev.yml +++ b/docker-compose.vhr18.dev.yml @@ -9,7 +9,12 @@ services: - type: bind source: ./data/ target: /data/ + ingestor: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev + fluentd: + image: registry.gitlab.eox.at/esa/prism/vs/fluentd:dev client: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:dev ports: - "80:80" configs: @@ -20,6 +25,7 @@ services: source: ./data/ target: /data/ renderer: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev ports: - "81:80" - "82:8080" @@ -28,6 +34,7 @@ services: source: ./data/ target: /data/ registrar: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev volumes: - type: bind source: ./data/ @@ -36,6 +43,7 @@ services: source: ./core/ target: /core/ cache: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_cache:dev ports: - "83:80" volumes: @@ -46,6 +54,7 @@ services: - source: mapcache-dev target: /mapcache-template.xml preprocessor: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:dev volumes: - type: tmpfs target: /tmp -- GitLab From 6c590ab8c8fac09e571e720dba0359fc7248283a Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Thu, 5 Nov 2020 10:15:05 +0100 Subject: [PATCH 2/3] update .gitlabci to see what happens in a branch --- .gitlab-ci.yml | 44 +++++++++++++------------------------------- 1 file changed, 13 insertions(+), 31 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6f9eddf1..8b9540d0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,28 +15,22 @@ build-master: script: - VERSION_1=`grep 'version="*"' core/Dockerfile | cut -d '"' -f2` - IMAGE_1="$CI_REGISTRY_IMAGE/pvs_core" - - docker pull "$IMAGE_1":latest || true - - docker build --cache-from "$IMAGE_1":latest -t "$IMAGE_1":latest -t "$IMAGE_1":$VERSION_1 core/ + - docker build -t "$IMAGE_1":dev -t "$IMAGE_1":$VERSION_1 core/ - VERSION_2=`grep 'version="*"' preprocessor/Dockerfile | cut -d '"' -f2` - IMAGE_2="$CI_REGISTRY_IMAGE/pvs_preprocessor" - - docker pull "$IMAGE_2":latest || true - - docker build --cache-from "$IMAGE_2":latest -t "$IMAGE_2":latest -t "$IMAGE_2":$VERSION_2 preprocessor/ + - docker build -t "$IMAGE_2":dev -t "$IMAGE_2":$VERSION_2 preprocessor/ - VERSION_3=`grep 'version="*"' client/Dockerfile | cut -d '"' -f2` - IMAGE_3="$CI_REGISTRY_IMAGE/pvs_client" - - docker pull "$IMAGE_3":latest || true - - docker build --cache-from "$IMAGE_3":latest -t "$IMAGE_3":latest -t "$IMAGE_3":$VERSION_3 client/ + - docker build -t "$IMAGE_3":dev -t "$IMAGE_3":$VERSION_3 client/ - VERSION_4=`grep 'version="*"' cache/Dockerfile | cut -d '"' -f2` - IMAGE_4="$CI_REGISTRY_IMAGE/pvs_cache" - - docker pull "$IMAGE_4":latest || true - - docker build --cache-from "$IMAGE_4":latest -t "$IMAGE_4":latest -t "$IMAGE_4":$VERSION_4 cache/ + - docker build -t "$IMAGE_4":dev -t "$IMAGE_4":$VERSION_4 cache/ - VERSION_5=`grep 'version="*"' fluentd/Dockerfile | cut -d '"' -f2` - IMAGE_5="$CI_REGISTRY_IMAGE/fluentd" - - docker pull "$IMAGE_5":latest || true - - docker build --cache-from "$IMAGE_5":latest -t "$IMAGE_5":latest -t "$IMAGE_5":$VERSION_5 fluentd/ + - docker build -t "$IMAGE_5":dev -t "$IMAGE_5":$VERSION_5 fluentd/ - VERSION_6=`grep 'version="*"' ingestor/Dockerfile | cut -d '"' -f2` - IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor" - - docker pull "$IMAGE_6":latest || true - - docker build --cache-from "$IMAGE_6":latest -t "$IMAGE_6":latest -t "$IMAGE_6":$VERSION_6 ingestor/ + - docker build -t "$IMAGE_6":dev -t "$IMAGE_6":$VERSION_6 ingestor/ - cd ./testing && ./gitlab_test.sh - if [ $? -ne 0 ]; then exit 1; fi # actually fail build - docker push "$IMAGE_1":$VERSION_1 @@ -53,7 +47,7 @@ build-master: - docker push "$IMAGE_6":latest only: - master - + - tags build: image: docker:latest stage: build @@ -63,29 +57,17 @@ build: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - IMAGE="$CI_REGISTRY_IMAGE/pvs_core" - - docker pull "$IMAGE":latest || true - - docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME core/ - - docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest" + - docker build -t "$IMAGE":dev core/ - IMAGE="$CI_REGISTRY_IMAGE/pvs_preprocessor" - - docker pull "$IMAGE":latest || true - - docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME preprocessor/ - - docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest" + - docker build -t "$IMAGE":dev preprocessor/ - IMAGE="$CI_REGISTRY_IMAGE/pvs_client" - - docker pull "$IMAGE":latest || true - - docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME client/ - - docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest" + - docker build -t "$IMAGE":dev client/ - IMAGE="$CI_REGISTRY_IMAGE/pvs_cache" - - docker pull "$IMAGE":latest || true - - docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME cache/ - - docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest" + - docker build -t "$IMAGE":dev cache/ - IMAGE="$CI_REGISTRY_IMAGE/fluentd" - - docker pull "$IMAGE":latest || true - - docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME fluentd/ - - docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest" + - docker build -t "$IMAGE":dev fluentd/ - IMAGE="$CI_REGISTRY_IMAGE/pvs_ingestor" - - docker pull "$IMAGE":latest || true - - docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME ingestor/ + - docker build -t "$IMAGE":dev ingestor/ - cd ./testing && ./gitlab_test.sh && cd - except: - master - -- GitLab From 52e510a913c6aeafcff8c4c0d7887d2edabdba3e Mon Sep 17 00:00:00 2001 From: Lubomir Bucek Date: Thu, 5 Nov 2020 10:46:41 +0100 Subject: [PATCH 3/3] revert removing pull & cache-from :latest to speedup --- .gitlab-ci.yml | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b9540d0..de2ece8a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,22 +15,28 @@ build-master: script: - VERSION_1=`grep 'version="*"' core/Dockerfile | cut -d '"' -f2` - IMAGE_1="$CI_REGISTRY_IMAGE/pvs_core" - - docker build -t "$IMAGE_1":dev -t "$IMAGE_1":$VERSION_1 core/ + - docker pull "$IMAGE_1":latest || true + - docker build --cache-from "$IMAGE_1":latest -t "$IMAGE_1":dev -t "$IMAGE_1":$VERSION_1 core/ - VERSION_2=`grep 'version="*"' preprocessor/Dockerfile | cut -d '"' -f2` - IMAGE_2="$CI_REGISTRY_IMAGE/pvs_preprocessor" - - docker build -t "$IMAGE_2":dev -t "$IMAGE_2":$VERSION_2 preprocessor/ + - docker pull "$IMAGE_2":latest || true + - docker build --cache-from "$IMAGE_2":latest -t "$IMAGE_2":dev -t "$IMAGE_2":$VERSION_2 preprocessor/ - VERSION_3=`grep 'version="*"' client/Dockerfile | cut -d '"' -f2` - IMAGE_3="$CI_REGISTRY_IMAGE/pvs_client" - - docker build -t "$IMAGE_3":dev -t "$IMAGE_3":$VERSION_3 client/ + - docker pull "$IMAGE_3":latest || true + - docker build --cache-from "$IMAGE_3":latest -t "$IMAGE_3":dev -t "$IMAGE_3":$VERSION_3 client/ - VERSION_4=`grep 'version="*"' cache/Dockerfile | cut -d '"' -f2` - IMAGE_4="$CI_REGISTRY_IMAGE/pvs_cache" - - docker build -t "$IMAGE_4":dev -t "$IMAGE_4":$VERSION_4 cache/ + - docker pull "$IMAGE_4":latest || true + - docker build --cache-from "$IMAGE_4":latest -t "$IMAGE_4":dev -t "$IMAGE_4":$VERSION_4 cache/ - VERSION_5=`grep 'version="*"' fluentd/Dockerfile | cut -d '"' -f2` - IMAGE_5="$CI_REGISTRY_IMAGE/fluentd" - - docker build -t "$IMAGE_5":dev -t "$IMAGE_5":$VERSION_5 fluentd/ + - docker pull "$IMAGE_5":latest || true + - docker build --cache-from "$IMAGE_5":latest -t "$IMAGE_5":dev -t "$IMAGE_5":$VERSION_5 fluentd/ - VERSION_6=`grep 'version="*"' ingestor/Dockerfile | cut -d '"' -f2` - IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor" - - docker build -t "$IMAGE_6":dev -t "$IMAGE_6":$VERSION_6 ingestor/ + - docker pull "$IMAGE_6":latest || true + - docker build --cache-from "$IMAGE_6":latest -t "$IMAGE_6":dev -t "$IMAGE_6":$VERSION_6 ingestor/ - cd ./testing && ./gitlab_test.sh - if [ $? -ne 0 ]; then exit 1; fi # actually fail build - docker push "$IMAGE_1":$VERSION_1 @@ -57,17 +63,24 @@ build: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY script: - IMAGE="$CI_REGISTRY_IMAGE/pvs_core" - - docker build -t "$IMAGE":dev core/ + - docker pull "$IMAGE":latest || true + - docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev core/ - IMAGE="$CI_REGISTRY_IMAGE/pvs_preprocessor" - - docker build -t "$IMAGE":dev preprocessor/ + - docker pull "$IMAGE":latest || true + - docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev preprocessor/ - IMAGE="$CI_REGISTRY_IMAGE/pvs_client" - - docker build -t "$IMAGE":dev client/ + - docker pull "$IMAGE":latest || true + - docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev client/ - IMAGE="$CI_REGISTRY_IMAGE/pvs_cache" - - docker build -t "$IMAGE":dev cache/ + - docker pull "$IMAGE":latest || true + - docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev cache/ - IMAGE="$CI_REGISTRY_IMAGE/fluentd" - - docker build -t "$IMAGE":dev fluentd/ + - docker pull "$IMAGE":latest || true + - docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev fluentd/ - IMAGE="$CI_REGISTRY_IMAGE/pvs_ingestor" - - docker build -t "$IMAGE":dev ingestor/ - - cd ./testing && ./gitlab_test.sh && cd - + - docker pull "$IMAGE":latest || true + - docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev ingestor/ + - cd ./testing && ./gitlab_test.sh + - if [ $? -ne 0 ]; then exit 1; fi # actually fail build except: - master -- GitLab