EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 323cc045 authored by Lubomir Dolezal's avatar Lubomir Dolezal
Browse files

Merge branch 'releases-rework' into 'master' 

Releases rework

See merge request !30
parents 671327c1 52e510a9
No related branches found
No related tags found
1 merge request!30Releases rework
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 15
− 20
View file @ 323cc045
......@@ -16,27 +16,27 @@ build-master:
- VERSION_1=`grep 'version="*"' core/Dockerfile | cut -d '"' -f2`
- IMAGE_1="$CI_REGISTRY_IMAGE/pvs_core"
- docker pull "$IMAGE_1":latest || true
- docker build --cache-from "$IMAGE_1":latest -t "$IMAGE_1":latest -t "$IMAGE_1":$VERSION_1 core/
- docker build --cache-from "$IMAGE_1":latest -t "$IMAGE_1":dev -t "$IMAGE_1":$VERSION_1 core/
- VERSION_2=`grep 'version="*"' preprocessor/Dockerfile | cut -d '"' -f2`
- IMAGE_2="$CI_REGISTRY_IMAGE/pvs_preprocessor"
- docker pull "$IMAGE_2":latest || true
- docker build --cache-from "$IMAGE_2":latest -t "$IMAGE_2":latest -t "$IMAGE_2":$VERSION_2 preprocessor/
- docker build --cache-from "$IMAGE_2":latest -t "$IMAGE_2":dev -t "$IMAGE_2":$VERSION_2 preprocessor/
- VERSION_3=`grep 'version="*"' client/Dockerfile | cut -d '"' -f2`
- IMAGE_3="$CI_REGISTRY_IMAGE/pvs_client"
- docker pull "$IMAGE_3":latest || true
- docker build --cache-from "$IMAGE_3":latest -t "$IMAGE_3":latest -t "$IMAGE_3":$VERSION_3 client/
- docker build --cache-from "$IMAGE_3":latest -t "$IMAGE_3":dev -t "$IMAGE_3":$VERSION_3 client/
- VERSION_4=`grep 'version="*"' cache/Dockerfile | cut -d '"' -f2`
- IMAGE_4="$CI_REGISTRY_IMAGE/pvs_cache"
- docker pull "$IMAGE_4":latest || true
- docker build --cache-from "$IMAGE_4":latest -t "$IMAGE_4":latest -t "$IMAGE_4":$VERSION_4 cache/
- docker build --cache-from "$IMAGE_4":latest -t "$IMAGE_4":dev -t "$IMAGE_4":$VERSION_4 cache/
- VERSION_5=`grep 'version="*"' fluentd/Dockerfile | cut -d '"' -f2`
- IMAGE_5="$CI_REGISTRY_IMAGE/fluentd"
- docker pull "$IMAGE_5":latest || true
- docker build --cache-from "$IMAGE_5":latest -t "$IMAGE_5":latest -t "$IMAGE_5":$VERSION_5 fluentd/
- docker build --cache-from "$IMAGE_5":latest -t "$IMAGE_5":dev -t "$IMAGE_5":$VERSION_5 fluentd/
- VERSION_6=`grep 'version="*"' ingestor/Dockerfile | cut -d '"' -f2`
- IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor"
- docker pull "$IMAGE_6":latest || true
- docker build --cache-from "$IMAGE_6":latest -t "$IMAGE_6":latest -t "$IMAGE_6":$VERSION_6 ingestor/
- docker build --cache-from "$IMAGE_6":latest -t "$IMAGE_6":dev -t "$IMAGE_6":$VERSION_6 ingestor/
- cd ./testing && ./gitlab_test.sh
- if [ $? -ne 0 ]; then exit 1; fi # actually fail build
- docker push "$IMAGE_1":$VERSION_1
......@@ -53,7 +53,7 @@ build-master:
- docker push "$IMAGE_6":latest
only:
- master
- tags
build:
image: docker:latest
stage: build
......@@ -64,28 +64,23 @@ build:
script:
- IMAGE="$CI_REGISTRY_IMAGE/pvs_core"
- docker pull "$IMAGE":latest || true
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME core/
- docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest"
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev core/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_preprocessor"
- docker pull "$IMAGE":latest || true
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME preprocessor/
- docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest"
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev preprocessor/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_client"
- docker pull "$IMAGE":latest || true
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME client/
- docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest"
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev client/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_cache"
- docker pull "$IMAGE":latest || true
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME cache/
- docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest"
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev cache/
- IMAGE="$CI_REGISTRY_IMAGE/fluentd"
- docker pull "$IMAGE":latest || true
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME fluentd/
- docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest"
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev fluentd/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_ingestor"
- docker pull "$IMAGE":latest || true
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME ingestor/
- cd ./testing && ./gitlab_test.sh && cd -
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":dev ingestor/
- cd ./testing && ./gitlab_test.sh
- if [ $? -ne 0 ]; then exit 1; fi # actually fail build
except:
- master
README.md
+ 17
− 18
View file @ 323cc045
......@@ -187,25 +187,17 @@ docker swarm init # initialize swarm
```
Build images:
Note we use **dev** tag for local development, so images need to be built locally
```
docker build core/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_core -t registry.gitlab.eox.at/esa/prism/vs/pvs_core
docker build cache/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_cache -t registry.gitlab.eox.at/esa/prism/vs/pvs_cache
docker build preprocessor/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor -t registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor
docker build client/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_client -t registry.gitlab.eox.at/esa/prism/vs/pvs_client
docker build fluentd/ --cache-from registry.gitlab.eox.at/esa/prism/vs/fluentd -t registry.gitlab.eox.at/esa/prism/vs/fluentd
docker build ingestor/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor -t registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor
```
Or pull them from the registry:
```
docker login -u {DOCKER_USER} -p {DOCKER_PASSWORD} registry.gitlab.eox.at
docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_core
docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_cache
docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor
docker pull registry.gitlab.eox.at/esa/prism/vs/pvs_client
docker pull registry.gitlab.eox.at/esa/prism/vs/fluentd
docker pull registry.gitlab.eox.at/esa/prism/vs/ingestor
docker build core/ -t registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev
docker build cache/ -t registry.gitlab.eox.at/esa/prism/vs/pvs_cache:dev
docker build preprocessor/ -t registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:dev
docker build client/ -t registry.gitlab.eox.at/esa/prism/vs/pvs_client:dev
docker build fluentd/ -t registry.gitlab.eox.at/esa/prism/vs/fluentd:dev
docker build ingestor/ -t registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev
```
For production deployment, as registry is open to public, this part is done by a later step `Deploy the stack in production` as it will pull necessary images automatically.
Create external network for stack to run:
```
docker network create -d overlay vhr18-extnet
......@@ -239,10 +231,17 @@ Deploy the stack in dev environment:
docker stack deploy -c docker-compose.vhr18.yml -c docker-compose.vhr18.dev.yml -c docker-compose.logging.yml -c docker-compose.logging.dev.yml vhr18-pvs # start VHR_IMAGE_2018 stack in dev mode, for example to use local sources
docker stack deploy -c docker-compose.emg.yml -c docker-compose.emg.dev.yml -c docker-compose.logging.yml -c docker-compose.logging.dev.yml emg-pvs # start Emergency stack in dev mode, for example to use local sources
```
Deploy base stack in production environment:
Deploy base & logging stack in production environment:
```
docker stack deploy -c docker-compose.base.ops.yml base-pvs
docker stack deploy -c docker-compose.logging.yml docker-compose.logging.ops.yml logging
```
Deploy the stack in production environment:
Please note that in order to reuse existing database volumes, <stack-name> needs to be the same. Here we use `vhr18-pvs` but in operational service `vhr18-pdas` is used.
```
docker stack deploy -c docker-compose.vhr18.yml -c docker-compose.vhr18.ops.yml vhr18-pvs
```
First steps:
```
# To register first data, use the following command inside the registrar container:
......
docker-compose.dem.dev.yml
+ 9
− 0
View file @ 323cc045
......@@ -9,7 +9,12 @@ services:
- type: bind
source: ./data/
target: /data/
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev
fluentd:
image: registry.gitlab.eox.at/esa/prism/vs/fluentd:dev
client:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:dev
ports:
- "80:80"
configs:
......@@ -20,6 +25,7 @@ services:
source: ./data/
target: /data/
renderer:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev
ports:
- "81:80"
- "82:8080"
......@@ -28,6 +34,7 @@ services:
source: ./data/
target: /data/
registrar:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev
volumes:
- type: bind
source: ./data/
......@@ -38,6 +45,7 @@ services:
logging:
driver: "fluentd"
cache:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_cache:dev
ports:
- "83:80"
volumes:
......@@ -48,6 +56,7 @@ services:
- source: mapcache-dev
target: /mapcache-template.xml
preprocessor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:dev
volumes:
- type: tmpfs
target: /tmp
......
docker-compose.emg.dev.yml
+ 9
− 0
View file @ 323cc045
......@@ -9,7 +9,12 @@ services:
- type: bind
source: ./data/
target: /data/
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev
fluentd:
image: registry.gitlab.eox.at/esa/prism/vs/fluentd:dev
client:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:dev
ports:
- "80:80"
configs:
......@@ -20,6 +25,7 @@ services:
source: ./data/
target: /data/
renderer:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev
ports:
- "81:80"
- "82:8080"
......@@ -28,6 +34,7 @@ services:
source: ./data/
target: /data/
registrar:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev
volumes:
- type: bind
source: ./data/
......@@ -36,6 +43,7 @@ services:
source: ./core/
target: /core/
cache:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_cache:dev
ports:
- "83:80"
volumes:
......@@ -46,6 +54,7 @@ services:
- source: mapcache-dev
target: /mapcache-template.xml
preprocessor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:dev
volumes:
- type: tmpfs
target: /tmp
......
docker-compose.vhr18.dev.yml
+ 9
− 0
View file @ 323cc045
......@@ -9,7 +9,12 @@ services:
- type: bind
source: ./data/
target: /data/
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev
fluentd:
image: registry.gitlab.eox.at/esa/prism/vs/fluentd:dev
client:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:dev
ports:
- "80:80"
configs:
......@@ -20,6 +25,7 @@ services:
source: ./data/
target: /data/
renderer:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev
ports:
- "81:80"
- "82:8080"
......@@ -28,6 +34,7 @@ services:
source: ./data/
target: /data/
registrar:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_core:dev
volumes:
- type: bind
source: ./data/
......@@ -36,6 +43,7 @@ services:
source: ./core/
target: /core/
cache:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_cache:dev
ports:
- "83:80"
volumes:
......@@ -46,6 +54,7 @@ services:
- source: mapcache-dev
target: /mapcache-template.xml
preprocessor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:dev
volumes:
- type: tmpfs
target: /tmp
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment