From 578875c3dbe5519f1c694b425a6a25760a64ff59 Mon Sep 17 00:00:00 2001
From: baloola <baloola-mu@hotmail.com>
Date: Tue, 24 Nov 2020 16:13:12 +0100
Subject: [PATCH] ingestor test
---
docker-compose.emg.dev.yml | 3 ++
testing/gitlab_test.sh | 27 +++++++++-
testing/ingestor_prep.py | 14 ++++++
testing/ingestor_test.py | 49 +++++++++++++++++++
...processor_test.sh => preprocessor_prep.sh} | 0
.../{registrar_test.sh => registrar_prep.sh} | 3 --
6 files changed, 91 insertions(+), 5 deletions(-)
create mode 100644 testing/ingestor_prep.py
create mode 100644 testing/ingestor_test.py
rename testing/{preprocessor_test.sh => preprocessor_prep.sh} (100%)
rename testing/{registrar_test.sh => registrar_prep.sh} (84%)
diff --git a/docker-compose.emg.dev.yml b/docker-compose.emg.dev.yml
index 672a23d5..13e9f119 100644
--- a/docker-compose.emg.dev.yml
+++ b/docker-compose.emg.dev.yml
@@ -11,6 +11,9 @@ services:
target: /data/
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:dev
+ environment:
+ INOTIFY_WATCH_DIR: "/mnt/data/"
+ REDIS_PREPROCESS_MD_QUEUE_KEY: "preprocess_queue"
fluentd:
image: registry.gitlab.eox.at/esa/prism/vs/fluentd:dev
client:
diff --git a/testing/gitlab_test.sh b/testing/gitlab_test.sh
index a3ef6f46..c7dd3fb7 100755
--- a/testing/gitlab_test.sh
+++ b/testing/gitlab_test.sh
@@ -33,14 +33,37 @@ printf $OS_PASSWORD | docker secret create OS_PASSWORD -
# create docker configs
printf $sftp_users_emg | docker config create sftp_users_emg -
+
+# override preprocessor default configuration file
+cat ./testing/testing_preprocessor_config.yml > ./config/emg_preprocessor-config.yml
+
+# deploy the stack
docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.yml emg-pvs
./docker-stack-wait.sh -n renderer -n registrar -n preprocessor -n database -n sftp emg-pvs
docker service ls
# perform the testing
-bash ./preprocessor_test.sh preprocessed_list.csv
-bash ./registrar_test.sh product_list.csv
+bash ./preprocessor_prep.sh preprocessed_list.csv
+bash ./registrar_prep.sh product_list.csv
+
+pytest registrar_test.py preprocessor_test.py renderer_test.py
+
+if [ $? -ne 0 ]; then echo "Pytest fail" >&2 && exit 1; fi
+
+swift delete preprocessor_results
+
+# ingest an item
+python3 ingestor_prep.py
+
+# wait for the process to complete
+sleep 90
+
+# test the existance of ingested item
+pytest ingestor_test.py
+
+swift delete preprocessor_results
+
if [ $? -ne 0 ]
then
echo "Failure in tests, logging from services:"
diff --git a/testing/ingestor_prep.py b/testing/ingestor_prep.py
new file mode 100644
index 00000000..1ae2d1f4
--- /dev/null
+++ b/testing/ingestor_prep.py
@@ -0,0 +1,14 @@
+import os
+
+import paramiko
+
+def sftp_connection():
+ username = os.environ['sftp_users_emg'].split(':')[0]
+ password = os.environ['sftp_users_emg'].split(':')[1]
+ transport = paramiko.Transport(('docker', 2223))
+ transport.connect(username=username, password=password)
+ sftp = paramiko.SFTPClient.from_transport(transport)
+ sftp.put('./testing/tpzonlineitem.xml', '/data/from/fepd/tpzonlineitem.xml')
+
+
+sftp_connection()
\ No newline at end of file
diff --git a/testing/ingestor_test.py b/testing/ingestor_test.py
new file mode 100644
index 00000000..219aec7c
--- /dev/null
+++ b/testing/ingestor_test.py
@@ -0,0 +1,49 @@
+import os
+import sys
+
+import pytest
+import subprocess
+
+from osgeo import gdal
+from swiftclient.service import SwiftService
+
+
+
+
+def set_gdal_swift_auth():
+ # parsing command line output of swift auth
+ auth_keys = subprocess.check_output(["swift", "auth"]).decode(sys.stdout.encoding).split("\n")
+ storage_url = auth_keys[0].split("OS_STORAGE_URL=")[1]
+ auth_token = auth_keys[1].split("OS_AUTH_TOKEN=")[1]
+ # setting gdal config
+ gdal.SetConfigOption("SWIFT_STORAGE_URL", storage_url)
+ gdal.SetConfigOption("SWIFT_AUTH_TOKEN", auth_token)
+
+@pytest.fixture
+def list_tifs(product):
+
+ preprocessed_list = []
+
+ with SwiftService() as swift:
+ # auth_options["prefix"] = product[0]
+ list_parts_gen = swift.list(
+ container='preprocessor_results', options= {"prefix": product[0]},
+ )
+ for page in list_parts_gen:
+ if page["success"]:
+ for item in page["listing"]:
+ if item["name"].endswith(".tif"):
+ preprocessed_list.append(item["name"])
+ return preprocessed_list
+
+def test_preprocessor(list_tifs):
+ # check if there are preprocessed results in the buckets
+ assert len(list_tifs) > 0
+ for item in list_tifs:
+ set_gdal_swift_auth()
+ image = gdal.Open('/vsiswift/preprocessor_results/%s' % item, gdal.GA_ReadOnly)
+ srcband = image.GetRasterBand(1)
+ assert srcband.Checksum() != None
+ assert srcband.Checksum() > 0
+
+
diff --git a/testing/preprocessor_test.sh b/testing/preprocessor_prep.sh
similarity index 100%
rename from testing/preprocessor_test.sh
rename to testing/preprocessor_prep.sh
diff --git a/testing/registrar_test.sh b/testing/registrar_prep.sh
similarity index 84%
rename from testing/registrar_test.sh
rename to testing/registrar_prep.sh
index 2c2ebc27..c3a2acec 100755
--- a/testing/registrar_test.sh
+++ b/testing/registrar_prep.sh
@@ -14,7 +14,4 @@ while read product; do
done < "$product_list_file"
-pytest
-if [ $? -ne 0 ]; then echo "Pytest fail" >&2 && exit 1; fi
-swift delete preprocessor_results
--
GitLab