EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 9d7649ba authored by Lubomir Dolezal's avatar Lubomir Dolezal
Browse files

Merge branch 'staging' into redis-statuses

parents 1481ce85 99bea2ff
No related branches found
No related tags found
2 merge requests!49Production release 1.1.1,!42Redis statuses
Hide whitespace changes
Inline Side-by-side
Showing
with 155 additions and 14 deletions
.gitlab-ci.yml
+ 12
− 1
View file @ 9d7649ba
......@@ -30,6 +30,9 @@ build-tag:
- IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor"
- docker pull "$IMAGE_6":latest || true
- docker build --cache-from "$IMAGE_6":latest -t "$IMAGE_6":dev -t "$IMAGE_6":$CI_COMMIT_TAG ingestor/
- IMAGE_7="$CI_REGISTRY_IMAGE/pvs_sftp"
- docker pull "$IMAGE_7":latest || true
- docker build --cache-from "$IMAGE_7":latest -t "$IMAGE_7":dev -t "$IMAGE_7":$CI_COMMIT_TAG sftp/
- cd ./testing && ./gitlab_test.sh
- if [ $? -ne 0 ]; then exit 1; fi # actually fail build
- docker push "$IMAGE_1":$CI_COMMIT_TAG
......@@ -38,6 +41,7 @@ build-tag:
- docker push "$IMAGE_4":$CI_COMMIT_TAG
- docker push "$IMAGE_5":$CI_COMMIT_TAG
- docker push "$IMAGE_6":$CI_COMMIT_TAG
- docker push "$IMAGE_7":$CI_COMMIT_TAG
only:
- tags
artifacts:
......@@ -65,11 +69,14 @@ build-master-staging:
- docker pull "$IMAGE_4":"$TAG_USED" || true
- docker build --cache-from "$IMAGE_4":"$TAG_USED" -t "$IMAGE_4":dev -t "$IMAGE_4":"$TAG_USED" cache/
- IMAGE_5="$CI_REGISTRY_IMAGE/fluentd"
- docker pull "$IMAGE_6":"$TAG_USED" || true
- docker pull "$IMAGE_5":"$TAG_USED" || true
- docker build --cache-from "$IMAGE_5":"$TAG_USED" -t "$IMAGE_5":dev -t "$IMAGE_5":"$TAG_USED" fluentd/
- IMAGE_6="$CI_REGISTRY_IMAGE/pvs_ingestor"
- docker pull "$IMAGE_6":"$TAG_USED" || true
- docker build --cache-from "$IMAGE_6":"$TAG_USED" -t "$IMAGE_6":dev -t "$IMAGE_6":"$TAG_USED" ingestor/
- IMAGE_7="$CI_REGISTRY_IMAGE/pvs_sftp"
- docker pull "$IMAGE_7":"$TAG_USED" || true
- docker build --cache-from "$IMAGE_7":latest -t "$IMAGE_7":dev -t "$IMAGE_7":"$TAG_USED" sftp/
- cd ./testing && ./gitlab_test.sh
- if [ $? -ne 0 ]; then exit 1; fi # actually fail build
- docker push "$IMAGE_1":"$TAG_USED"
......@@ -78,6 +85,7 @@ build-master-staging:
- docker push "$IMAGE_4":"$TAG_USED"
- docker push "$IMAGE_5":"$TAG_USED"
- docker push "$IMAGE_6":"$TAG_USED"
- docker push "$IMAGE_7":"$TAG_USED"
only:
- staging
- master
......@@ -112,6 +120,9 @@ build:
- IMAGE="$CI_REGISTRY_IMAGE/pvs_ingestor"
- docker pull "$IMAGE":staging || true
- docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev ingestor/
- IMAGE="$CI_REGISTRY_IMAGE/pvs_sftp"
- docker pull "$IMAGE":staging || true
- docker build --cache-from "$IMAGE":staging -t "$IMAGE":dev sftp/
- cd ./testing && ./gitlab_test.sh
- if [ $? -ne 0 ]; then exit 1; fi # actually fail build
except:
......
core/registrar/backend.py
+ 3
− 2
View file @ 9d7649ba
......@@ -211,12 +211,13 @@ class EOxServerBackend(Backend):
# register masks
for mask_identifier, mask_type_name in mapping.get('masks', {}).items():
_, mask_item = source.get_container_and_path(item.mask_files.get(mask_identifier))
mask_item = item.mask_files.get(mask_identifier)
if mask_item:
_, mask_path = source.get_container_and_path()
logger.info(f"Adding mask (file) {mask_type_name} to product")
MaskRegistrator().register(
product.identifier,
storage + [mask_item],
storage + [mask_path],
mask_type_name,
)
......
docker-compose.dem.dev.yml
+ 2
− 0
View file @ 9d7649ba
......@@ -9,6 +9,8 @@ services:
- type: bind
source: ./data/
target: /data/
sftp:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_sftp:dev
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev
fluentd:
......
docker-compose.dem.ops.yml
+ 12
− 2
View file @ 9d7649ba
......@@ -122,6 +122,18 @@ services:
placement:
constraints:
- node.labels.type == internal
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:release-1.0.0 # bumpversion
environment:
REDIS_PREPROCESS_MD_QUEUE_KEY: "preprocess_queue"
deploy:
placement:
constraints: [node.role == manager]
sftp:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_sftp:release-1.0.0 # bumpversion
deploy:
placement:
constraints: [node.role == manager]
client:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:release-1.0.0 # bumpversion
configs:
......@@ -225,8 +237,6 @@ services:
target: /etc/shibboleth/shibd.logger
- source: native-logger
target: /etc/shibboleth/native.logger
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:release-1.0.0 # bumpversion
networks:
extnet:
name: dem-extnet
......
docker-compose.dem.yml
+ 9
− 1
View file @ 9d7649ba
......@@ -170,7 +170,7 @@ services:
deploy:
replicas: 1
sftp:
image: atmoz/sftp:latest
image: registry.gitlab.eox.at/esa/prism/vs/pvs_sftp:latest
volumes:
- type: volume
source: report-data
......@@ -187,6 +187,14 @@ services:
replicas: 1
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
env_file:
- env/dem_redis.env
command:
["python3", "/filedaemon.py"]
volumes:
- type: volume
source: from-fepd
target: /mnt/data
environment:
INSTANCE_ID: "prism-view-server_ingestor"
deploy:
......
docker-compose.emg.dev.yml
+ 2
− 0
View file @ 9d7649ba
......@@ -9,6 +9,8 @@ services:
- type: bind
source: ./data/
target: /data/
sftp:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_sftp:dev
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev
fluentd:
......
docker-compose.emg.ops.yml
+ 12
− 2
View file @ 9d7649ba
......@@ -122,6 +122,18 @@ services:
placement:
constraints:
- node.labels.type == internal
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:release-1.0.0 # bumpversion
environment:
REDIS_PREPROCESS_MD_QUEUE_KEY: "preprocess_queue"
deploy:
placement:
constraints: [node.role == manager]
sftp:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_sftp:release-1.0.0 # bumpversion
deploy:
placement:
constraints: [node.role == manager]
client:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:release-1.0.0 # bumpversion
configs:
......@@ -225,8 +237,6 @@ services:
target: /etc/shibboleth/shibd.logger
- source: native-logger
target: /etc/shibboleth/native.logger
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:release-1.0.0 # bumpversion
networks:
extnet:
name: emg-extnet
......
docker-compose.emg.yml
+ 9
− 1
View file @ 9d7649ba
......@@ -106,6 +106,14 @@ services:
["/run-seeder.sh"]
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
command:
["python3", "/filedaemon.py"]
volumes:
- type: volume
source: from-fepd
target: /mnt/data
env_file:
- env/emg_redis.env
environment:
INSTANCE_ID: "prism-view-server_ingestor"
deploy:
......@@ -177,7 +185,7 @@ services:
deploy:
replicas: 1
sftp:
image: atmoz/sftp:latest
image: registry.gitlab.eox.at/esa/prism/vs/pvs_sftp:latest
volumes:
- type: volume
source: report-data
......
docker-compose.vhr18.dev.yml
+ 2
− 0
View file @ 9d7649ba
......@@ -9,6 +9,8 @@ services:
- type: bind
source: ./data/
target: /data/
sftp:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_sftp:dev
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev
fluentd:
......
docker-compose.vhr18.ops.yml
+ 12
− 2
View file @ 9d7649ba
......@@ -122,6 +122,18 @@ services:
placement:
constraints:
- node.labels.type == internal
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:release-1.0.0 # bumpversion
environment:
REDIS_PREPROCESS_MD_QUEUE_KEY: "preprocess_queue"
deploy:
placement:
constraints: [node.role == manager]
sftp:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_sftp:release-1.0.0 # bumpversion
deploy:
placement:
constraints: [node.role == manager]
client:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:release-1.0.0 # bumpversion
configs:
......@@ -225,8 +237,6 @@ services:
target: /etc/shibboleth/shibd.logger
- source: native-logger
target: /etc/shibboleth/native.logger
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:release-1.0.0 # bumpversion
networks:
extnet:
name: vhr18-extnet
......
docker-compose.vhr18.yml
+ 9
− 1
View file @ 9d7649ba
......@@ -110,6 +110,14 @@ services:
["/run-seeder.sh"]
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
command:
["python3", "/filedaemon.py"]
volumes:
- type: volume
source: from-fepd
target: /mnt/data
env_file:
- env/vhr18_redis.env
environment:
INSTANCE_ID: "prism-view-server_ingestor"
deploy:
......@@ -181,7 +189,7 @@ services:
deploy:
replicas: 1
sftp:
image: atmoz/sftp:latest
image: registry.gitlab.eox.at/esa/prism/vs/pvs_sftp:latest
volumes:
- type: volume
source: report-data
......
sftp/Dockerfile 0 → 100644
+ 40
− 0
View file @ 9d7649ba
#------------------------------------------------------------------------------
#
# Project: prism view server
# Authors: Stephan Meissl <stephan.meissl@eox.at>
#
#------------------------------------------------------------------------------
# Copyright (C) 2020 EOX IT Services GmbH <https://eox.at>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies of this Software or works derived from this Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
FROM atmoz/sftp:latest
LABEL name="prism view server docker base image" \
vendor="EOX IT Services GmbH <https://eox.at>" \
license="MIT Copyright (C) 2020 EOX IT Services GmbH <https://eox.at>" \
type="prism view server docker base image" \
version="1.0.0"
RUN mkdir /etc/sftp.d
ADD entrypoint.sh /etc/sftp.d
RUN chmod -v +x /etc/sftp.d/entrypoint.sh
sftp/entrypoint.sh 0 → 100644
+ 12
− 0
View file @ 9d7649ba
#!/bin/bash
echo " baloola"
for user_home in /home/* ; do
echo $user_home
if [ -d "$user_home" ]; then
username=`basename $user_home`
echo "Setup $user_home/data folder for $username"
mkdir -p $user_home/data
chown -R $username:users $user_home/data
fi
done
testing/preprocessor_test.py
+ 2
− 1
View file @ 9d7649ba
......@@ -43,7 +43,8 @@ def list_tifs(products):
return preprocessed_list
def test_preprocessor(list_tifs):
# check if there are preprocessed results in the buckets
assert len(list_tifs) > 0
for item in list_tifs:
set_gdal_swift_auth()
image = gdal.Open('/vsiswift/preprocessor_results/%s' % item, gdal.GA_ReadOnly)
......
testing/renderer_test.py
+ 5
− 1
View file @ 9d7649ba
......@@ -44,7 +44,9 @@ def test_renderer(identifiers):
# wms succsess
assert wms_response.status_code == 200
# check if there are registerd products
assert len(wms_items) > 0
def test_wcs(identifiers):
wcs_items = get_requests('http://docker:81/opensearch/collections/Emergency/atom/', [], 'wcs', 'GetCoverage')
......@@ -68,4 +70,6 @@ def test_wcs(identifiers):
assert wcs_response.status_code == 200
assert srcband.Checksum() != None
assert srcband.Checksum() > 0
# check if there are registerd products
assert len(wcs_items) > 0
testing/tpzonlineitem.xml 0 → 100644
+ 12
− 0
View file @ 9d7649ba
<?xml version="1.0" encoding="UTF-8"?>
<!--Sample XML file generated by XMLSpy v2013 sp1 (http://www.altova.com)-->
<OnLineArchiveItem xsi:schemaLocation="http://www.telespazio.com/CSCDA/CDD/PDAS PDAS_interfaces%2020190924_1916.xsd" xmlns="http://www.telespazio.com/CSCDA/CDD/PDAS" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<identifier>urn:eop:KGS:K3_20160101113014_19339_02281264_L0F_28E2
</identifier>
<path>
<Service>OS</Service>
<URL>KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar
</URL>
</path>
<status>added</status>
</OnLineArchiveItem>
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment