diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2d2a6d117389615cee55e5c39b0a7fba55f9fcec..b6361546bdf49b72793bfb2cae69eff559790d0f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,12 @@
variables:
DOCKER_TLS_CERTDIR: ""
+stages:
+ - build
+ - test
+ - publish
+
+
build-master:
image: docker:latest
stage: build
@@ -66,3 +72,38 @@ build:
- docker push "$IMAGE:$CI_COMMIT_REF_NAME"
except:
- master
+test:
+ image: docker:latest
+
+ services:
+ - docker:dind
+ before_script:
+ - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+ - chmod +x env_setup.sh wait_for_container.sh
+ - ./env_setup.sh
+ - mkdir data
+ - docker build core/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_core -t registry.gitlab.eox.at/esa/prism/vs/pvs_core:latest
+ - docker build cache/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_cache -t registry.gitlab.eox.at/esa/prism/vs/pvs_cache:latest
+ - docker build preprocessor/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor -t registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:latest
+ - docker build client/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_client -t registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest
+ - docker push registry.gitlab.eox.at/esa/prism/vs/pvs_core:latest
+ - docker push registry.gitlab.eox.at/esa/prism/vs/pvs_cache:latest
+ - docker push registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:latest
+ - docker push registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest
+ - docker swarm init
+ - docker network create -d overlay emg-extnet
+ - docker stack deploy -c docker-compose.emg.yml -c docker-compose.emg.dev.yml emg-pvs
+ script:
+
+ - apk update && apk add postgresql-dev gcc python3-dev musl-dev py-pip gdal
+ - pip3 install -r requirements.txt
+ - ./wait_for_container.sh
+ - docker exec -i $(docker ps -qf "name=emg-pvs_registrar") python3 /var/www/pvs/dev/pvs_instance/manage.py storage create pvs_testing pvs_testing --type swift --storage-auth auth-cloud-ovh
+ - docker exec -i $(docker ps -qf "name=emg-pvs_registrar") python3 /core/registrar.py --objects-prefix "OA/PH1B/0.1/b9/urn:eop:PHR:MULTISPECTRAL_0.5m:DS_PHR1B_201608070959189_FR1_PX_E012N32_0719_00974_4148/0000/PH1B_PHR_FUS_1A_20160807T095918_20160807T095920_TOU_1234_4148.DIMA.tar"
+ - pytest -s registrar_test.py --name OA/PH1B/0.1/b9/urn:eop:PHR:MULTISPECTRAL_0.5m:DS_PHR1B_201608070959189_FR1_PX_E012N32_0719_00974_4148/0000/PH1B_PHR_FUS_1A_20160807T095918_20160807T095920_TOU_1234_4148.DIMA.tar
+
+
+
+ after_script:
+
+
\ No newline at end of file
diff --git a/conftest.py b/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..7a1d3249386440525e74a35d4574a36f9781c633
--- /dev/null
+++ b/conftest.py
@@ -0,0 +1,10 @@
+def pytest_addoption(parser):
+ parser.addoption("--name", action="store", default="default name")
+
+
+def pytest_generate_tests(metafunc):
+ # This is called for every test. Only get/set command line arguments
+ # if the argument is specified in the list of test "fixturenames".
+ option_value = metafunc.config.option.name
+ if 'name' in metafunc.fixturenames and option_value is not None:
+ metafunc.parametrize("name", [option_value])
\ No newline at end of file
diff --git a/core/registrar.py b/core/registrar.py
index 962e474819e081d08db302dd50180f5de1583158..45b60237a64abfc2ccd0f356b36c79ed1090c6f0 100644
--- a/core/registrar.py
+++ b/core/registrar.py
@@ -143,6 +143,7 @@ def registrar(
container=upload_container, options={"prefix": objects_prefix},
)
for page in list_parts_gen:
+ print(page)
if page["success"]:
for item in page["listing"]:
if item["name"].endswith(".xml"):
@@ -154,7 +155,7 @@ def registrar(
elif item["name"].endswith(".vrt"):
data_package = item["name"]
has_vrt = True
- else:
+ elif not item["name"].endswith(".tar"):
raise RegistrationError(
"Product with objects prefix '%s' has "
"wrong content '%s'."
@@ -315,7 +316,7 @@ if __name__ == "__main__":
setup_logging(arg_values.verbosity)
- collection = os.environ.get('Collection')
+ collection = os.environ.get('COLLECTION')
if collection is None:
logger.critical("Collection environment variable not set.")
sys.exit(1)
diff --git a/docker-compose.emg.dev.yml b/docker-compose.emg.dev.yml
index c0cf7b5fe871f6016e0ea485e9238e1a5a229973..34264cade2b10ce59e6371f59433f91404fd989a 100644
--- a/docker-compose.emg.dev.yml
+++ b/docker-compose.emg.dev.yml
@@ -1,5 +1,10 @@
version: "3.6"
services:
+ database:
+ networks:
+ - extnet
+ ports:
+ - "5432:5432"
client:
ports:
- "80:80"
@@ -39,3 +44,7 @@ services:
- type: bind
source: ./preprocessor/
target: /preprocessor/
+networks:
+ extnet:
+ name: emg-extnet
+ external: true
\ No newline at end of file
diff --git a/env_setup.sh b/env_setup.sh
new file mode 100644
index 0000000000000000000000000000000000000000..9f2bb95bc7ffe5d2cb9e26b7676bf2c464ba304d
--- /dev/null
+++ b/env_setup.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+cat $vhr18_db > ./env/vhr18_db.env
+cat $vhr18_django > ./env/vhr18_django.env
+cat $vhr18_obs > ./env/vhr18_obs.env
+
+cat $emg_db > ./env/emg_db.env
+cat $emg_django > ./env/emg_django.env
+cat $emg_obs > ./env/emg_obs.env
+
+
+set -o allexport
+
+source ./env/emg_db.env
+source ./env/vhr18_db.env
+
+set +o allexport
+
+
+sed -i -e 's/emg-data/pvs_testing/g' ./env/emg.env
+sed -i -e 's/vhr18-data/pvs_testing/g' ./env/vhr18.env
+
+sed -i -e 's/emg-cache/pvs_testing/g' ./env/emg_obs.env
+sed -i -e 's/vhr18-cache/pvs_testing/g' ./env/vhr18_obs.env
diff --git a/preprocessor/preprocessor.py b/preprocessor/preprocessor.py
index 9894bbad077b66162a1648fd62f5818ac664f08f..e24892eb0b93fe92632d9fc00ae3a38339b80a6d 100644
--- a/preprocessor/preprocessor.py
+++ b/preprocessor/preprocessor.py
@@ -349,6 +349,12 @@ if __name__ == "__main__":
"Path to object holding tar archive file of product."
)
)
+ parser.add_argument(
+ "--upload-container", default=None,
+ help=(
+ "The name of the swift container where the result is uploaded."
+ )
+ )
parser.add_argument(
"--replace", action="store_true",
help=(
@@ -380,15 +386,17 @@ if __name__ == "__main__":
setup_logging(arg_values.verbosity)
- collection = os.environ.get('Collection')
+ collection = os.environ.get('COLLECTION')
if collection is None:
logger.critical("Collection environment variable not set.")
sys.exit(1)
- upload_container = os.environ.get('UPLOAD_CONTAINER')
+ upload_container = arg_values.upload_container
if upload_container is None:
- logger.critical("UPLOAD_CONTAINER environment variable not set.")
- sys.exit(1)
+ upload_container = os.environ.get('UPLOAD_CONTAINER')
+ if upload_container is None:
+ logger.critical("UPLOAD_CONTAINER environment variable not set.")
+ sys.exit(1)
if arg_values.mode == "standard":
preprocessor(
diff --git a/registrar_test.py b/registrar_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..43cc696132ab02e1b785d339c2c5983665656e03
--- /dev/null
+++ b/registrar_test.py
@@ -0,0 +1,44 @@
+import psycopg2
+import os
+
+
+with open('./env/emg_db.env', 'r') as f:
+ env = dict(
+ line.split('=', 1)
+ for line in f
+ )
+database= env['DB_NAME'].replace('\n','')
+port = env['DB_PORT'].replace('\n','')
+host = env['DB_HOST'].replace('\n','')
+database_password= env['DB_PW'].replace('\n','')
+database_user = env['DB_USER'].replace('\n','')
+
+
+def connect_to_db(eo_id):
+ global db_name, coverage_id
+ connection= None
+ try:
+ connection = psycopg2.connect(dbname=database, user=database_user, password=database_password, host='docker', port=port)
+ cursor = connection.cursor()
+ db_name = connection.get_dsn_parameters()["dbname"]
+ postgreSQL_select_Query = "SELECT identifier FROM coverages_eoobject WHERE identifier = '%s';" % eo_id
+ cursor.execute(postgreSQL_select_Query)
+ coverage_id = cursor.fetchone()[0]
+
+ except (Exception, psycopg2.Error) as error :
+ print ("Error while connecting to PostgreSQL", error)
+ finally:
+ #closing database connection.
+ if connection:
+ cursor.close()
+ connection.close()
+ print("PostgreSQL connection is closed")
+
+
+
+def test_db_name(name):
+ identifier = name.split('/')[4]
+ connect_to_db(identifier)
+ assert coverage_id == identifier
+ assert db_name == database
+
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..56ba698664ddb78df20e57a7c6159d1da82feba9
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,4 @@
+pytest
+psycopg2
+# python-swiftclient
+# python-keystoneclient
\ No newline at end of file
diff --git a/wait_for_container.sh b/wait_for_container.sh
new file mode 100755
index 0000000000000000000000000000000000000000..e4a7569f03a33fd18106fede2cbb54afec0cfe8a
--- /dev/null
+++ b/wait_for_container.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+# waiting for the database to finish migration!
+sleep 60
+
+while [ -z $(docker ps -qf "name=emg-pvs_renderer") ]; do
+ >&2 echo "Container is unavailable - sleeping"
+ sleep 15
+ done