EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit a1dfb683 authored by Lubomir Dolezal's avatar Lubomir Dolezal
Browse files

checking for other status codes than 0,1 in manage.py id check 

test on emg now
parent 681d68bf
Branches
Tags
1 merge request!49Production release 1.1.1
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1306 additions and 1319 deletions
.gitlab-ci.yml
+ 19
− 15
View file @ a1dfb683
...@@ -44,9 +44,6 @@ build-tag: ...@@ -44,9 +44,6 @@ build-tag:
- docker push "$IMAGE_7":$CI_COMMIT_TAG - docker push "$IMAGE_7":$CI_COMMIT_TAG
only: only:
- tags - tags
artifacts:
paths:
- /tmp/*.log
build-master-staging: build-master-staging:
image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
stage: build stage: build
...@@ -92,8 +89,13 @@ build-master-staging: ...@@ -92,8 +89,13 @@ build-master-staging:
except: except:
- tags - tags
artifacts: artifacts:
when: always
expire_in: 4 week
paths: paths:
- /tmp/*.log - emg-pvs_database.txt
- emg-pvs_preprocessor.txt
- emg-pvs_registrar.txt
- emg-pvs_renderer.txt
build: build:
image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest image: registry.gitlab.eox.at/esa/prism/vs/docker-base-testing:latest
stage: build stage: build
...@@ -130,15 +132,17 @@ build: ...@@ -130,15 +132,17 @@ build:
- staging - staging
- master - master
artifacts: artifacts:
when: always
expire_in: 4 week
paths: paths:
# - /tmp/emg-pvs_cache # - emg-pvs_cache.txt
# - /tmp/emg-pvs_client # - emg-pvs_client.txt
- /tmp/emg-pvs_database - emg-pvs_database.txt
# - /tmp/emg-pvs_fluentd # - emg-pvs_fluentd.txt
# - /tmp/emg-pvs_ingestor # - emg-pvs_ingestor.txt
- /tmp/emg-pvs_preprocessor - emg-pvs_preprocessor.txt
# - /tmp/emg-pvs_redis # - emg-pvs_redis.txt
- /tmp/emg-pvs_registrar - emg-pvs_registrar.txt
- /tmp/emg-pvs_renderer - emg-pvs_renderer.txt
# - /tmp/emg-pvs_seeder # - emg-pvs_seeder.txt
# - /tmp/emg-pvs_sftp # - emg-pvs_sftp.txt
config/dem_init-db.sh
+ 67
− 67
View file @ a1dfb683
# Check if collection exits in database and initialize database only if not # Check if collection exits in database and initialize database only if not
if python3 manage.py id check "${COLLECTION}"; then # on command fail restart service
echo "Initialize database" function initialize {
echo "Initialize coverage types"
python3 manage.py coveragetype import /rgbnir_definition.json --traceback python3 manage.py coveragetype import /rgbnir_definition.json --traceback
echo "Initializing collection '${COLLECTION}'."
if [ "${COLLECTION}" == "DEM" ]; then python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED --traceback \
echo "Initializing collection '${COLLECTION}'." --coverage-type "int16_grayscale"
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED --traceback \ --red "gray" \
--coverage-type "int16_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED --traceback \
--red-range -100 4000 \ --coverage-type "int16_grayscale"
--red-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED --traceback \ --red "gray" \
--coverage-type "int16_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED --traceback \
--red-range -100 4000 \ --coverage-type "float32_grayscale"
--red-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED --traceback \ --red "gray" \
--coverage-type "float32_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP --traceback \
--red-range -100 4000 \ --coverage-type "float32_grayscale"
--red-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP --traceback \ --red "gray" \
--coverage-type "float32_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED --traceback \
--red-range -100 4000 \ --coverage-type "float32_grayscale"
--red-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED --traceback \ --red "gray" \
--coverage-type "float32_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED --traceback \
--red-range -100 4000 \ --coverage-type "float32_grayscale"
--red-nodata 0 python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED --traceback \
python3 manage.py producttype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED --traceback \ --red "gray" \
--coverage-type "float32_grayscale" --red-range -100 4000 \
python3 manage.py browsetype create "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED --traceback \ --red-nodata 0
--red "gray" \ python3 manage.py collectiontype create "${COLLECTION}"_Collection --traceback \
--red-range -100 4000 \ --coverage-type "float32_grayscale" \
--red-nodata 0 --coverage-type "int16_grayscale" \
--product-type "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED \
python3 manage.py collectiontype create "${COLLECTION}"_Collection --traceback \ --product-type "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED \
--coverage-type "float32_grayscale" \ --product-type "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED \
--coverage-type "int16_grayscale" \ --product-type "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP \
--product-type "${COLLECTION}"_Product_COP-DEM_GLO-30-DTED \ --product-type "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED \
--product-type "${COLLECTION}"_Product_COP-DEM_GLO-90-DTED \ --product-type "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED
--product-type "${COLLECTION}"_Product_COP-DEM_EEA-10-DGED \ python3 manage.py collection create "${COLLECTION}" --type "${COLLECTION}"_Collection --traceback
--product-type "${COLLECTION}"_Product_COP-DEM_EEA-10-INSP \ python3 manage.py collection create "${COLLECTION}_COP-DEM_EEA-10-DGED" --type "${COLLECTION}"_Collection --traceback
--product-type "${COLLECTION}"_Product_COP-DEM_GLO-30-DGED \ python3 manage.py collection create "${COLLECTION}_COP-DEM_EEA-10-INSP" --type "${COLLECTION}"_Collection --traceback
--product-type "${COLLECTION}"_Product_COP-DEM_GLO-90-DGED python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-30-DGED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-30-DTED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}" --type "${COLLECTION}"_Collection --traceback python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-90-DGED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_EEA-10-DGED" --type "${COLLECTION}"_Collection --traceback python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-90-DTED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_EEA-10-INSP" --type "${COLLECTION}"_Collection --traceback }
python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-30-DGED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-30-DTED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-90-DGED" --type "${COLLECTION}"_Collection --traceback
python3 manage.py collection create "${COLLECTION}_COP-DEM_GLO-90-DTED" --type "${COLLECTION}"_Collection --traceback
else
echo "Provided collection '${COLLECTION}' not valid."
fi
id_check=$(python3 manage.py id check "${COLLECTION}")
if [[ $id_check == *"is already in use by a 'Collection'"* ]]; then
# name is already taken
echo "Using existing database"
elif [[ $id_check == *"is currently not in use"* ]]; then
initialize
else else
echo "Using existing database" echo "Restarting service inside init-db by emitting an error"
set -eo pipefail
false
fi fi
config/emg_init-db.sh
+ 803
− 801
View file @ a1dfb683
This diff is collapsed.
config/vhr18_init-db.sh
+ 391
− 402
View file @ a1dfb683
This diff is collapsed.
core/entrypoint.sh
+ 1
− 1
View file @ a1dfb683
...@@ -40,4 +40,4 @@ EOF ...@@ -40,4 +40,4 @@ EOF
file_env "OS_PASSWORD" file_env "OS_PASSWORD"
file_env "DJANGO_PASSWORD" file_env "DJANGO_PASSWORD"
eval "/opt/eoxserver/eoxserver-entrypoint.sh $@" eval "/opt/eoxserver/eoxserver-entrypoint.sh $@" >&2
docker-compose.dem.yml
+ 3
− 1
View file @ a1dfb683
...@@ -43,13 +43,14 @@ services: ...@@ -43,13 +43,14 @@ services:
- env/dem_obs.env - env/dem_obs.env
secrets: secrets:
- DJANGO_PASSWORD - DJANGO_PASSWORD
- OS_PASSWORD - OS_PASSWORD
environment: environment:
INSTANCE_ID: "prism-view-server_renderer" INSTANCE_ID: "prism-view-server_renderer"
INSTALL_DIR: "/var/www/pvs/dev/" INSTALL_DIR: "/var/www/pvs/dev/"
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "database:5432" WAIT_SERVICES: "database:5432"
WAIT_TIMEOUT: 60
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
configs: configs:
...@@ -151,6 +152,7 @@ services: ...@@ -151,6 +152,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "redis:6379 database:5432" WAIT_SERVICES: "redis:6379 database:5432"
WAIT_TIMEOUT: 60
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
REPORTING_DIR: "/mnt/reports/" REPORTING_DIR: "/mnt/reports/"
......
docker-compose.emg.yml
+ 2
− 0
View file @ a1dfb683
...@@ -50,6 +50,7 @@ services: ...@@ -50,6 +50,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "database:5432" WAIT_SERVICES: "database:5432"
WAIT_TIMEOUT: 60
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
configs: configs:
...@@ -165,6 +166,7 @@ services: ...@@ -165,6 +166,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "redis:6379 database:5432" WAIT_SERVICES: "redis:6379 database:5432"
WAIT_TIMEOUT: 60
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
REPORTING_DIR: "/mnt/reports/" REPORTING_DIR: "/mnt/reports/"
......
docker-compose.vhr18.yml
+ 2
− 0
View file @ a1dfb683
...@@ -50,6 +50,7 @@ services: ...@@ -50,6 +50,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "database:5432" WAIT_SERVICES: "database:5432"
WAIT_TIMEOUT: 60
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
configs: configs:
...@@ -168,6 +169,7 @@ services: ...@@ -168,6 +169,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh" INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh" STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "redis:6379 database:5432" WAIT_SERVICES: "redis:6379 database:5432"
WAIT_TIMEOUT: 60
OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD" OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD" DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
REPORTING_DIR: "/mnt/reports/" REPORTING_DIR: "/mnt/reports/"
......
testing/gitlab_test.sh
+ 11
− 23
View file @ a1dfb683
#!/bin/sh #!/bin/sh
any_test_failed=1 # false
function log_services () {
if [ $? -ne 0 ]
then
echo "Failure in tests, logging from services:"
for service in $(docker service ls --format "{{.Name}}"); do
docker service ps $service --no-trunc >> "/tmp/$service.log"
docker service logs $service >> "/tmp/$service.log"
docker service ps $service --no-trunc
docker service logs $service
done
exit 1
fi
}
# fetch secrets and write them to their according files # fetch secrets and write them to their according files
cat $vhr18_db > ../env/vhr18_db.env cat $vhr18_db > ../env/vhr18_db.env
cat $vhr18_django > ../env/vhr18_django.env cat $vhr18_django > ../env/vhr18_django.env
...@@ -53,29 +40,30 @@ cat ./testing_preprocessor_config.yml>../config/emg_preprocessor-config.yml ...@@ -53,29 +40,30 @@ cat ./testing_preprocessor_config.yml>../config/emg_preprocessor-config.yml
# deploy the stack # deploy the stack
docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.yml emg-pvs docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.yml emg-pvs
./docker-stack-wait.sh -n renderer -n registrar -n preprocessor -n database -n sftp emg-pvs ./docker-stack-wait.sh -n renderer -n registrar -n preprocessor -n database -n sftp -n ingestor emg-pvs
docker service ls
# perform the testing # perform the testing
swift delete preprocessor_results
bash ./preprocessor_prep.sh preprocessed_list.csv bash ./preprocessor_prep.sh preprocessed_list.csv
bash ./registrar_prep.sh product_list.csv bash ./registrar_prep.sh product_list.csv
pytest registrar_test.py preprocessor_test.py renderer_test.py pytest registrar_test.py preprocessor_test.py renderer_test.py
if [ $? -ne 0 ]; then any_test_failed=0;fi
log_services
swift delete preprocessor_results swift delete preprocessor_results
# ingest an item # ingest an item
python3 ingestor_prep.py python3 ingestor_prep.py
# wait for the process to complete # wait for the process to complete
sleep 90 sleep 60
# test the existance of ingested item # test the existance of ingested item
pytest ingestor_test.py pytest ingestor_test.py
if [ $? -ne 0 ]; then any_test_failed=0;fi
log_services for service in $(docker service ls --format "{{.Name}}"); do
docker service ps $service --no-trunc >> "$CI_PROJECT_DIR/$service.txt"
docker service logs $service --timestamps >> "$CI_PROJECT_DIR/$service.txt"
done
swift delete preprocessor_results if [ $any_test_failed -ne 1 ]; then echo "Some test failed"; exit 1;fi
\ No newline at end of file
testing/tpzonlineitem.xml
+ 7
− 9
View file @ a1dfb683
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!--Sample XML file generated by XMLSpy v2013 sp1 (http://www.altova.com)--> <!--Sample XML file generated by XMLSpy v2013 sp1 (http://www.altova.com)-->
<OnLineArchiveItem xsi:schemaLocation="http://www.telespazio.com/CSCDA/CDD/PDAS PDAS_interfaces%2020190924_1916.xsd" xmlns="http://www.telespazio.com/CSCDA/CDD/PDAS" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <OnLineArchiveItem xmlns="http://www.telespazio.com/CSCDA/CDD/PDAS" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.telespazio.com/CSCDA/CDD/PDAS PDAS_interfaces%2020190924_1916.xsd">
<identifier>urn:eop:KGS:K3_20160101113014_19339_02281264_L0F_28E2 <identifier>urn:eop:KGS:K3_20160101113014_19339_02281264_L0F_28E2</identifier>
</identifier> <path>
<path> <Service>OS</Service>
<Service>OS</Service> <URL>KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar</URL>
<URL>KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar </path>
</URL> <status>added</status>
</path>
<status>added</status>
</OnLineArchiveItem> </OnLineArchiveItem>
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment