Merge branch 'ingestor_test' into 'staging'

Ingestor test

See merge request !44

docker-compose.emg.dev.yml

@@ -13,6 +13,9 @@ services:
     image: registry.gitlab.eox.at/esa/prism/vs/pvs_sftp:dev
   ingestor:
     image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev
+    environment:
+      INOTIFY_WATCH_DIR: "/mnt/data/"
+      REDIS_PREPROCESS_MD_QUEUE_KEY: "preprocess_queue"
   fluentd:
     image: registry.gitlab.eox.at/esa/prism/vs/fluentd:dev
   client:

testing/gitlab_test.sh

@@ -33,14 +33,37 @@ printf $OS_PASSWORD | docker secret create OS_PASSWORD -
 
 # create docker configs
 printf $sftp_users_emg | docker config create sftp_users_emg -
+
+# override preprocessor default configuration file
+cat ./testing_preprocessor_config.yml>../config/emg_preprocessor-config.yml
+
+# deploy the stack
 docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.yml emg-pvs
 
 ./docker-stack-wait.sh -n renderer -n registrar -n preprocessor -n database -n sftp emg-pvs
 
 docker service ls
 # perform the testing
-bash ./preprocessor_test.sh preprocessed_list.csv
-bash ./registrar_test.sh product_list.csv
+bash ./preprocessor_prep.sh preprocessed_list.csv
+bash ./registrar_prep.sh product_list.csv
+
+pytest registrar_test.py preprocessor_test.py renderer_test.py
+
+if [ $? -ne 0 ]; then echo "Pytest fail" >&2 && exit 1; fi
+
+swift delete preprocessor_results
+
+# ingest an item
+python3 ingestor_prep.py
+
+# wait for the process to complete
+sleep 90
+
+# test the existance of ingested item
+pytest ingestor_test.py
+
+swift delete preprocessor_results
+
 if [ $? -ne 0 ]
 then
   echo "Failure in tests, logging from services:"

testing/ingestor_prep.py

+import os
+
+import paramiko
+
+def sftp_connection():
+    username = os.environ['sftp_users_emg'].split(':')[0]
+    password = os.environ['sftp_users_emg'].split(':')[1]
+    transport = paramiko.Transport(('docker', 2223))
+    transport.connect(username=username, password=password)
+    sftp = paramiko.SFTPClient.from_transport(transport)
+    sftp.put('./tpzonlineitem.xml', '/data/from/fepd/tpzonlineitem.xml')
+    report_list = sftp.listdir('data/from/fepd/')
+    print("Sucssesfully adding %s to the sftp directory" % report_list)
+
+
+sftp_connection()
\ No newline at end of file

testing/ingestor_test.py

+import os
+import sys
+
+import pytest
+import subprocess
+
+from osgeo import gdal
+from swiftclient.service import SwiftService
+
+
+
+
+def set_gdal_swift_auth():
+    # parsing command line output of swift auth
+    auth_keys = subprocess.check_output(["swift", "auth"]).decode(sys.stdout.encoding).split("\n")
+    storage_url = auth_keys[0].split("OS_STORAGE_URL=")[1]
+    auth_token = auth_keys[1].split("OS_AUTH_TOKEN=")[1]
+    # setting gdal config
+    gdal.SetConfigOption("SWIFT_STORAGE_URL", storage_url)
+    gdal.SetConfigOption("SWIFT_AUTH_TOKEN", auth_token)
+
+@pytest.fixture
+def list_tifs(product='KS03_AIS_PSH_1G_20160101T113020_20160101T113022_KGS_019339_75CE.tar'):
+
+    preprocessed_list = []
+
+    with SwiftService() as swift:
+        # auth_options["prefix"] = product[0]
+        list_parts_gen = swift.list(
+            container='preprocessor_results', options= {"prefix": product[0]},
+        )
+        for page in list_parts_gen:
+            if page["success"]:
+                for item in page["listing"]:
+                    if item["name"].endswith(".tif"):
+                        preprocessed_list.append(item["name"])
+    return preprocessed_list
+
+def test_preprocessor(list_tifs):
+    # check if there are preprocessed results in the buckets 
+    assert len(list_tifs) > 0
+    for item in list_tifs:
+        set_gdal_swift_auth()
+        image = gdal.Open('/vsiswift/preprocessor_results/%s' % item, gdal.GA_ReadOnly)
+        srcband = image.GetRasterBand(1)
+        assert srcband.Checksum() != None
+        assert srcband.Checksum() > 0 
+
+

testing/registrar_test.sh → testing/registrar_prep.sh

@@ -14,7 +14,4 @@ while read product; do
 
 done < "$product_list_file"
 
-pytest
-if [ $? -ne 0 ]; then echo "Pytest fail" >&2 && exit 1; fi
 
-swift delete preprocessor_results