EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 46172b3d authored by Lubomir Dolezal's avatar Lubomir Dolezal
Browse files

Merge branch 'testing-builds' into 'staging' 

Testing builds

See merge request !40
parents 681d68bf a1dfb683
No related branches found
No related tags found
2 merge requests!49Production release 1.1.1,!40Testing builds
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1306 additions and 1319 deletions
.gitlab-ci.yml
+ 19
− 15
View file @ 46172b3d
...@@ -44,9 +44,6 @@ build-tag: ...@@ -44,9 +44,6 @@ build-tag:
- docker push "$IMAGE_7":$CI_COMMIT_TAG - docker push "$IMAGE_7":$CI_COMMIT_TAG
only: only:
- tags - tags
artifacts:
paths:
- /tmp/*.log
build-master-staging: build-master-staging:
image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
stage: build stage: build
...@@ -92,8 +89,13 @@ build-master-staging: ...@@ -92,8 +89,13 @@ build-master-staging:
except: except:
- tags - tags
artifacts: artifacts:
when: always
expire_in: 4 week
paths: paths:
- /tmp/*.log - emg-pvs_database.txt
- emg-pvs_preprocessor.txt
- emg-pvs_registrar.txt
- emg-pvs_renderer.txt
build: build:
image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
stage: build stage: build
...@@ -130,15 +132,17 @@ build: ...@@ -130,15 +132,17 @@ build:
- staging - staging
- master - master
artifacts: artifacts:
when: always
expire_in: 4 week
paths: paths:
# - /tmp/emg-pvs_cache # - emg-pvs_cache.txt
# - /tmp/emg-pvs_client # - emg-pvs_client.txt
- /tmp/emg-pvs_database - emg-pvs_database.txt
# - /tmp/emg-pvs_fluentd # - emg-pvs_fluentd.txt
# - /tmp/emg-pvs_ingestor # - emg-pvs_ingestor.txt
- /tmp/emg-pvs_preprocessor - emg-pvs_preprocessor.txt
# - /tmp/emg-pvs_redis # - emg-pvs_redis.txt
- /tmp/emg-pvs_registrar - emg-pvs_registrar.txt
- /tmp/emg-pvs_renderer - emg-pvs_renderer.txt
# - /tmp/emg-pvs_seeder # - emg-pvs_seeder.txt
# - /tmp/emg-pvs_sftp # - emg-pvs_sftp.txt
config/dem_init-db.sh
+ 67
− 67
View file @ 46172b3d
# Check if collection exits in database and initialize database only if not # Check if collection exits in database and initialize database only if not
if python3 manage.py id check "${COLLECTION}"; then # on command fail restart service
echo "Initialize database" function initialize {
echo "Initialize coverage types"
python3 manage.py coveragetype import /rgbnir_definition.json --traceback python3 manage.py coveragetype import /rgbnir_definition.json --traceback
echo "Initializing collection '${COLLECTION}'."
if [ "${COLLECTION}" == "DEM" ]; then python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED --traceback \
echo "Initializing collection '${COLLECTION}'." --coverage-type "int16_grayscale"
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED --traceback \ --red "gray" \
--coverage-type "int16_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED --traceback \
--red-range -100 4000 \ --coverage-type "int16_grayscale"
--red-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED --traceback \ --red "gray" \
--coverage-type "int16_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED --traceback \
--red-range -100 4000 \ --coverage-type "float32_grayscale"
--red-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED --traceback \ --red "gray" \
--coverage-type "float32_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP --traceback \
--red-range -100 4000 \ --coverage-type "float32_grayscale"
--red-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP --traceback \ --red "gray" \
--coverage-type "float32_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED --traceback \
--red-range -100 4000 \ --coverage-type "float32_grayscale"
--red-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED --traceback \ --red "gray" \
--coverage-type "float32_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED --traceback \
--red-range -100 4000 \ --coverage-type "float32_grayscale"
--red-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED --traceback \ --red "gray" \
--coverage-type "float32_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py collectiontype create "${COLLECTION}"_Collection --traceback \
--red-range -100 4000 \ --coverage-type "float32_grayscale" \
--red-nodata 0 --coverage-type "int16_grayscale" \
--product-type "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED \
python3 manage.py collectiontype create "${COLLECTION}"_Collection --traceback \ --product-type "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED \
--coverage-type "float32_grayscale" \ --product-type "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED \
--coverage-type "int16_grayscale" \ --product-type "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP \
--product-type "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED \ --product-type "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED \
--product-type "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED \ --product-type "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED
--product-type "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED \ python3 manage.py collection create "${COLLECTION}" --type "${COLLECTION}"_Collection --traceback
--product-type "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP \ python3 manage.py collection create "${COLLECTION}_COP-DEM_EEA-10-DGED" --type "${COLLECTION}"_Collection --traceback
--product-type "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED \ python3 manage.py collection create "${COLLECTION}_COP-DEM_EEA-10-INSP" --type "${COLLECTION}"_Collection --traceback
--product-type "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-30-DGED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-30-DTED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}" --type "${COLLECTION}"_Collection --traceback python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-90-DGED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_EEA-10-DGED" --type "${COLLECTION}"_Collection --traceback python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-90-DTED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_EEA-10-INSP" --type "${COLLECTION}"_Collection --traceback }
python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-30-DGED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-30-DTED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-90-DGED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-90-DTED" --type "${COLLECTION}"_Collection --traceback
else
echo "Provided collection '${COLLECTION}' not valid."
fi
id_check=$(python3 manage.py id check "${COLLECTION}")
if [[ $id_check == *"is already in use by a 'Collection'"* ]]; then
# name is already taken
echo "Using existing database"
elif [[ $id_check == *"is currently not in use"* ]]; then
initialize
else else
echo "Using existing database" echo "Restarting service inside init-db by emitting an error"
set -eo pipefail
false
fi fi
config/emg_init-db.sh
+ 803
− 801
View file @ 46172b3d
This diff is collapsed.
config/vhr18_init-db.sh
+ 391
− 402
View file @ 46172b3d
This diff is collapsed.
core/entrypoint.sh
+ 1
− 1
View file @ 46172b3d
...@@ -40,4 +40,4 @@ EOF ...@@ -40,4 +40,4 @@ EOF
file_env "OS_PASSWORD" file_env "OS_PASSWORD"
file_env "DJANGO_PASSWORD" file_env "DJANGO_PASSWORD"
eval "/opt/eoxserver/eoxserver-entrypoint.sh $@" eval "/opt/eoxserver/eoxserver-entrypoint.sh $@" >&2
docker-compose.dem.yml
+ 3
− 1
View file @ 46172b3d
...@@ -43,13 +43,14 @@ services: ...@@ -43,13 +43,14 @@ services:
- env/dem_obs.env - env/dem_obs.env
secrets: secrets:
- DJANGO_PASSWORD - DJANGO_PASSWORD
- OS_PASSWORD - OS_PASSWORD
environment: environment:
INSTANCE_ID: "prism-view-server_renderer" INSTANCE_ID: "prism-view-server_renderer"
INSTALL_DIR: "/var/www/pvs/dev/" INSTALL_DIR: "/var/www/pvs/dev/"
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "database:5432" WAIT_SERVICES: "database:5432"
WAIT_TIMEOUT: 60
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
configs: configs:
...@@ -151,6 +152,7 @@ services: ...@@ -151,6 +152,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "redis:6379 database:5432" WAIT_SERVICES: "redis:6379 database:5432"
WAIT_TIMEOUT: 60
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
REPORTING_DIR: "/mnt/reports/" REPORTING_DIR: "/mnt/reports/"
......
docker-compose.emg.yml
+ 2
− 0
View file @ 46172b3d
...@@ -50,6 +50,7 @@ services: ...@@ -50,6 +50,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "database:5432" WAIT_SERVICES: "database:5432"
WAIT_TIMEOUT: 60
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
configs: configs:
...@@ -165,6 +166,7 @@ services: ...@@ -165,6 +166,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "redis:6379 database:5432" WAIT_SERVICES: "redis:6379 database:5432"
WAIT_TIMEOUT: 60
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
REPORTING_DIR: "/mnt/reports/" REPORTING_DIR: "/mnt/reports/"
......
docker-compose.vhr18.yml
+ 2
− 0
View file @ 46172b3d
...@@ -50,6 +50,7 @@ services: ...@@ -50,6 +50,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "database:5432" WAIT_SERVICES: "database:5432"
WAIT_TIMEOUT: 60
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
configs: configs:
...@@ -168,6 +169,7 @@ services: ...@@ -168,6 +169,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "redis:6379 database:5432" WAIT_SERVICES: "redis:6379 database:5432"
WAIT_TIMEOUT: 60
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
REPORTING_DIR: "/mnt/reports/" REPORTING_DIR: "/mnt/reports/"
......
testing/gitlab_test.sh
+ 11
− 23
View file @ 46172b3d
#!/bin/sh #!/bin/sh
any_test_failed=1 # false
function log_services () {
if [ $? -ne 0 ]
then
echo "Failure in tests, logging from services:"
for service in $(docker service ls --format "{{.Name}}"); do
docker service ps $service --no-trunc >> "/tmp/$service.log"
docker service logs $service >> "/tmp/$service.log"
docker service ps $service --no-trunc
docker service logs $service
done
exit 1
fi
}
# fetch secrets and write them to their according files # fetch secrets and write them to their according files
cat $vhr18_db > ../env/vhr18_db.env cat $vhr18_db > ../env/vhr18_db.env
cat $vhr18_django > ../env/vhr18_django.env cat $vhr18_django > ../env/vhr18_django.env
...@@ -53,29 +40,30 @@ cat ./testing_preprocessor_config.yml>../config/emg_preprocessor-config.yml ...@@ -53,29 +40,30 @@ cat ./testing_preprocessor_config.yml>../config/emg_preprocessor-config.yml
# deploy the stack # deploy the stack
docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.yml emg-pvs docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.yml emg-pvs
./docker-stack-wait.sh -n renderer -n registrar -n preprocessor -n database -n sftp emg-pvs ./docker-stack-wait.sh -n renderer -n registrar -n preprocessor -n database -n sftp -n ingestor emg-pvs
docker service ls
# perform the testing # perform the testing
swift delete preprocessor_results
bash ./preprocessor_prep.sh preprocessed_list.csv bash ./preprocessor_prep.sh preprocessed_list.csv
bash ./registrar_prep.sh product_list.csv bash ./registrar_prep.sh product_list.csv
pytest registrar_test.py preprocessor_test.py renderer_test.py pytest registrar_test.py preprocessor_test.py renderer_test.py
if [ $? -ne 0 ]; then any_test_failed=0;fi
log_services
swift delete preprocessor_results swift delete preprocessor_results
# ingest an item # ingest an item
python3 ingestor_prep.py python3 ingestor_prep.py
# wait for the process to complete # wait for the process to complete
sleep 90 sleep 60
# test the existance of ingested item # test the existance of ingested item
pytest ingestor_test.py pytest ingestor_test.py
if [ $? -ne 0 ]; then any_test_failed=0;fi
log_services for service in $(docker service ls --format "{{.Name}}"); do
docker service ps $service --no-trunc >> "$CI_PROJECT_DIR/$service.txt"
docker service logs $service --timestamps >> "$CI_PROJECT_DIR/$service.txt"
done
swift delete preprocessor_results if [ $any_test_failed -ne 1 ]; then echo "Some test failed"; exit 1;fi
\ No newline at end of file
testing/tpzonlineitem.xml
+ 7
− 9
View file @ 46172b3d
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!--Sample XML file generated by XMLSpy v2013 sp1 (http://www.altova.com)--> <!--Sample XML file generated by XMLSpy v2013 sp1 (http://www.altova.com)-->
<OnLineArchiveItem xsi:schemaLocation="http://www.telespazio.com/CSCDA/CDD/PDAS PDAS_interfaces%2020190924_1916.xsd" xmlns="http://www.telespazio.com/CSCDA/CDD/PDAS" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <OnLineArchiveItem xmlns="http://www.telespazio.com/CSCDA/CDD/PDAS" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.telespazio.com/CSCDA/CDD/PDAS PDAS_interfaces%2020190924_1916.xsd">
<identifier>urn:eop:KGS:K3_20160101113014_19339_02281264_L0F_28E2 <identifier>urn:eop:KGS:K3_20160101113014_19339_02281264_L0F_28E2</identifier>
</identifier> <path>
<path> <Service>OS</Service>
<Service>OS</Service> <URL>KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar</URL>
<URL>KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar </path>
</URL> <status>added</status>
</path>
<status>added</status>
</OnLineArchiveItem> </OnLineArchiveItem>
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment