Merge branch 'testing_branch' into 'master'

Testing branch

See merge request !5

.gitlab-ci.yml

 variables:
   DOCKER_TLS_CERTDIR: ""
 
+stages:
+  - build
+  - test
+  - publish
+
+
 build-master:
   image: docker:latest
   stage: build
@@ -66,3 +72,38 @@ build:
     - docker push "$IMAGE:$CI_COMMIT_REF_NAME"
   except:
     - master
+test:
+  image: docker:latest
+
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+    - chmod +x env_setup.sh wait_for_container.sh
+    - ./env_setup.sh
+    - mkdir data
+    - docker build core/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_core -t registry.gitlab.eox.at/esa/prism/vs/pvs_core:latest
+    - docker build cache/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_cache -t registry.gitlab.eox.at/esa/prism/vs/pvs_cache:latest
+    - docker build preprocessor/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor -t registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:latest
+    - docker build client/ --cache-from registry.gitlab.eox.at/esa/prism/vs/pvs_client -t registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest
+    - docker push registry.gitlab.eox.at/esa/prism/vs/pvs_core:latest
+    - docker push registry.gitlab.eox.at/esa/prism/vs/pvs_cache:latest
+    - docker push registry.gitlab.eox.at/esa/prism/vs/pvs_preprocessor:latest
+    - docker push registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest
+    - docker swarm init
+    - docker network create -d overlay emg-extnet
+    - docker stack deploy -c docker-compose.emg.yml -c docker-compose.emg.dev.yml emg-pvs
+  script:
+
+    - apk update && apk add postgresql-dev gcc python3-dev musl-dev py-pip gdal
+    - pip3 install -r requirements.txt
+    - ./wait_for_container.sh
+    - docker exec -i $(docker ps -qf "name=emg-pvs_registrar") python3 /var/www/pvs/dev/pvs_instance/manage.py storage create pvs_testing pvs_testing --type swift --storage-auth auth-cloud-ovh
+    - docker exec -i $(docker ps -qf "name=emg-pvs_registrar") python3 /core/registrar.py --objects-prefix "OA/PH1B/0.1/b9/urn:eop:PHR:MULTISPECTRAL_0.5m:DS_PHR1B_201608070959189_FR1_PX_E012N32_0719_00974_4148/0000/PH1B_PHR_FUS_1A_20160807T095918_20160807T095920_TOU_1234_4148.DIMA.tar"
+    - pytest -s registrar_test.py --name OA/PH1B/0.1/b9/urn:eop:PHR:MULTISPECTRAL_0.5m:DS_PHR1B_201608070959189_FR1_PX_E012N32_0719_00974_4148/0000/PH1B_PHR_FUS_1A_20160807T095918_20160807T095920_TOU_1234_4148.DIMA.tar
+    
+
+
+  after_script:
+
+        
\ No newline at end of file

conftest.py

+def pytest_addoption(parser):
+    parser.addoption("--name", action="store", default="default name")
+
+
+def pytest_generate_tests(metafunc):
+    # This is called for every test. Only get/set command line arguments
+    # if the argument is specified in the list of test "fixturenames".
+    option_value = metafunc.config.option.name
+    if 'name' in metafunc.fixturenames and option_value is not None:
+        metafunc.parametrize("name", [option_value])
\ No newline at end of file

core/registrar.py

@@ -143,6 +143,7 @@ def registrar(
             container=upload_container, options={"prefix": objects_prefix},
         )
         for page in list_parts_gen:
+            print(page)
             if page["success"]:
                 for item in page["listing"]:
                     if item["name"].endswith(".xml"):
@@ -154,7 +155,7 @@ def registrar(
                     elif item["name"].endswith(".vrt"):
                         data_package = item["name"]
                         has_vrt = True
-                    else:
+                    elif not item["name"].endswith(".tar"):
                         raise RegistrationError(
                             "Product with objects prefix '%s' has "
                             "wrong content '%s'."
@@ -315,7 +316,7 @@ if __name__ == "__main__":
 
     setup_logging(arg_values.verbosity)
 
-    collection = os.environ.get('Collection')
+    collection = os.environ.get('COLLECTION')
     if collection is None:
         logger.critical("Collection environment variable not set.")
         sys.exit(1)

docker-compose.emg.dev.yml

 version: "3.6"
 services:
+  database:
+    networks:
+      - extnet
+    ports:
+      - "5432:5432"
   client:
     ports:
       - "80:80"
@@ -39,3 +44,7 @@ services:
       - type: bind
         source: ./preprocessor/
         target: /preprocessor/
+networks:
+  extnet:
+    name: emg-extnet
+    external: true
\ No newline at end of file

env_setup.sh

+#!/bin/sh
+cat $vhr18_db > ./env/vhr18_db.env
+cat $vhr18_django > ./env/vhr18_django.env
+cat $vhr18_obs > ./env/vhr18_obs.env
+
+cat $emg_db > ./env/emg_db.env
+cat $emg_django > ./env/emg_django.env
+cat $emg_obs > ./env/emg_obs.env
+
+
+set -o allexport
+
+source ./env/emg_db.env
+source ./env/vhr18_db.env
+
+set +o allexport
+
+
+sed -i -e 's/emg-data/pvs_testing/g' ./env/emg.env
+sed -i -e 's/vhr18-data/pvs_testing/g' ./env/vhr18.env
+
+sed -i -e 's/emg-cache/pvs_testing/g' ./env/emg_obs.env
+sed -i -e 's/vhr18-cache/pvs_testing/g' ./env/vhr18_obs.env

preprocessor/preprocessor.py

@@ -349,6 +349,12 @@ if __name__ == "__main__":
             "Path to object holding tar archive file of product."
         )
     )
+    parser.add_argument(
+        "--upload-container", default=None,
+        help=(
+            "The name of the swift container where the result is uploaded."
+        )
+    )
     parser.add_argument(
         "--replace", action="store_true",
         help=(
@@ -380,15 +386,17 @@ if __name__ == "__main__":
 
     setup_logging(arg_values.verbosity)
 
-    collection = os.environ.get('Collection')
+    collection = os.environ.get('COLLECTION')
     if collection is None:
         logger.critical("Collection environment variable not set.")
         sys.exit(1)
 
-    upload_container = os.environ.get('UPLOAD_CONTAINER')
+    upload_container = arg_values.upload_container
     if upload_container is None:
-        logger.critical("UPLOAD_CONTAINER environment variable not set.")
-        sys.exit(1)
+        upload_container = os.environ.get('UPLOAD_CONTAINER')
+        if upload_container is None:
+            logger.critical("UPLOAD_CONTAINER environment variable not set.")
+            sys.exit(1)
 
     if arg_values.mode == "standard":
         preprocessor(

registrar_test.py

+import psycopg2
+import os
+
+
+with open('./env/emg_db.env', 'r') as f:
+    env = dict(
+        line.split('=', 1)
+        for line in f
+    )
+database= env['DB_NAME'].replace('\n','')
+port = env['DB_PORT'].replace('\n','')
+host = env['DB_HOST'].replace('\n','')
+database_password= env['DB_PW'].replace('\n','')
+database_user = env['DB_USER'].replace('\n','')
+
+
+def connect_to_db(eo_id):
+    global db_name, coverage_id
+    connection= None
+    try:
+        connection = psycopg2.connect(dbname=database, user=database_user, password=database_password, host='docker', port=port)
+        cursor = connection.cursor()
+        db_name = connection.get_dsn_parameters()["dbname"]
+        postgreSQL_select_Query = "SELECT identifier FROM coverages_eoobject WHERE identifier = '%s';" % eo_id
+        cursor.execute(postgreSQL_select_Query)
+        coverage_id = cursor.fetchone()[0]
+
+    except (Exception, psycopg2.Error) as error :
+        print ("Error while connecting to PostgreSQL", error)
+    finally:
+        #closing database connection.
+        if connection:
+            cursor.close()
+            connection.close()
+            print("PostgreSQL connection is closed")
+                
+
+
+def test_db_name(name):
+    identifier = name.split('/')[4]
+    connect_to_db(identifier)
+    assert coverage_id == identifier
+    assert db_name == database
+

requirements.txt

+pytest
+psycopg2
+# python-swiftclient
+# python-keystoneclient
\ No newline at end of file

wait_for_container.sh

+#!/bin/sh
+# waiting for the database to finish migration!
+sleep 60
+
+while [ -z $(docker ps -qf "name=emg-pvs_renderer") ]; do
+    >&2 echo "Container is unavailable - sleeping"
+    sleep 15
+  done