Merge branch 'sftp_logs' into 'master'

Sftp logs

See merge request !18

README.md

@@ -222,6 +222,20 @@ Select `@timestamp` as time field
 ([see also](https://www.elastic.co/guide/en/kibana/current/tutorial-define-index.html)).
 
 
+### setup sftp
+
+The `SFTP` image allow remote access into 2 logging folders, you can define (edit/add) users, passwords and (UID/GID) in the respictive configuration file ( e.g  *config/vhr_sftp_users.conf* ). 
+
+The default username is `eox`, once the stack is deployed you can sftp into the logging folders through port 2222 on -if you rn the dev stack- localhost :
+
+```bash
+sftp -P 2222 eox@127.0.0.1
+``` 
+You will log in  into`/home/eox/data` directory which contains the 2 logging directories : `to/panda` and `from/fepd`
+
+ **NOTE:**  The mounted directory that you are directed into is *`/home/user`*, where `user` is the username, hence when changing the username in the `.conf` file, the `sftp` mounted volumes path in `docker-compse.<collection>.yml` must change respectivly.
+ 
+
 # Documentation
 
 ## Installation

config/dem_sftp_users.conf

+***REMOVED***:1001:100
\ No newline at end of file

config/emg_sftp_users.conf

+***REMOVED***:1001:100
\ No newline at end of file

config/vhr18_sftp_users.conf

+***REMOVED***:1001:100
\ No newline at end of file

docker-compose.dem.yml

@@ -148,6 +148,23 @@ services:
     image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest
     deploy:
       replicas: 1
+  sftp:
+    image: atmoz/sftp:latest
+    volumes:
+      - type: volume
+        source: report-data
+        target: /home/eox/data/to/panda
+      - type: volume
+        source: from-fepd
+        target: /home/eox/data/from/fepd
+    configs:
+      - source: sftp-users
+        target: /etc/sftp/users.conf
+
+    ports:
+        - "2222:22"
+    deploy:
+      replicas: 1
   ingestor:
     image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
     env_file:
@@ -159,6 +176,8 @@ services:
     networks:
       - intnet
 configs:
+  sftp-users:
+    file: ./config/dem_sftp_users.conf
   init-db:
     file: ./config/dem_init-db.sh
   mapcache-dev:
@@ -175,6 +194,7 @@ volumes:
   db-data:
   redis-data:
   instance-data:
+  from-fepd:
   report-data:
 networks:
   intnet:

docker-compose.emg.yml

@@ -158,7 +158,26 @@ services:
     image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest
     deploy:
       replicas: 1
+  sftp:
+    image: atmoz/sftp:latest
+    volumes:
+      - type: volume
+        source: report-data
+        target: /home/eox/data/to/panda
+      - type: volume
+        source: from-fepd
+        target: /home/eox/data/from/fepd
+    configs:
+      - source: sftp-users
+        target: /etc/sftp/users.conf
+
+    ports:
+        - "2222:22"
+    deploy:
+      replicas: 1
 configs:
+  sftp-users:
+    file: ./config/emg_sftp_users.conf
   init-db:
     file: ./config/emg_init-db.sh
   mapcache-dev:
@@ -175,6 +194,7 @@ volumes:
   db-data:
   redis-data:
   instance-data:
+  from-fepd:
   report-data:
 networks:
   intnet:

docker-compose.vhr18.yml

@@ -161,7 +161,30 @@ services:
     image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest
     deploy:
       replicas: 1
+  sftp:
+    image: atmoz/sftp:latest
+    volumes:
+      - type: volume
+        source: report-data
+        target: /home/eox/data/to/panda
+      - type: volume
+        source: from-fepd
+        target: /home/eox/data/from/fepd
+    configs:
+      - source: sftp-users
+        target: /etc/sftp/users.conf
+    deploy:
+      replicas: 1
+
+    ports:
+        - "2222:22"
+  ingestor:
+    image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
+    deploy:
+      replicas: 1
 configs:
+  sftp-users:
+    file: ./config/vhr18_sftp_users.conf
   init-db:
     file: ./config/vhr18_init-db.sh
   mapcache-dev:
@@ -178,6 +201,7 @@ volumes:
   db-data:
   redis-data:
   instance-data:
+  from-fepd:
   report-data:
 networks:
   intnet:

testing/gitlab_test.sh

@@ -27,7 +27,7 @@ mkdir data
 docker swarm init
 docker network create -d overlay emg-extnet
 docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.yml emg-pvs
-apk update && apk add bash postgresql-dev gcc python3-dev musl-dev py-pip gdal
+apk update && apk add bash postgresql-dev gcc python3-dev musl-dev py-pip gdal libffi-dev openssl-dev make
 pip3 install -r requirements.txt
 
 ./docker-stack-wait.sh -n renderer -n registrar -n preprocessor emg-pvs

testing/registrar_test.py

@@ -3,7 +3,9 @@ import csv
 
 import pytest
 import psycopg2
+import paramiko
 from dotenv import load_dotenv
+from xml.etree import ElementTree
 
 
 @pytest.fixture(scope="session")
@@ -28,6 +30,14 @@ def identifiers():
         yield csv.reader(f)
 
 
+@pytest.fixture
+def sftp_connection():
+
+    transport = paramiko.Transport(('docker',2222))
+    transport.connect(username='eox', password='password')
+    with paramiko.SFTPClient.from_transport(transport) as sftp:
+        yield sftp
+
 def query_eo_object(connection, eo_id):
     query = f"SELECT identifier FROM coverages_eoobject WHERE identifier = '{eo_id}';"
     with connection.cursor() as cursor:
@@ -39,3 +49,34 @@ def test_db_name(connection, identifiers):
     for row in identifiers:
         identifier = row[0].split('/')[4]
         query_eo_object(connection, identifier)
+
+
+def compare_links(sftp, product_xml, product):
+
+    report= sftp.file("data/to/panda/%s" % product_xml)
+
+
+    xml_file= report.read()
+    root = ElementTree.fromstring(xml_file.decode('utf-8').strip())
+
+    urls = root.findall('{http://www.telespazio.com/CSCDA/CDD/PDAS}URL')
+    wms_link = urls[0].find('{http://www.telespazio.com/CSCDA/CDD/PDAS}URL').text
+    wcs_link = urls[1].find('{http://www.telespazio.com/CSCDA/CDD/PDAS}URL').text
+
+    wms_capabilities = 'emg.pass.copernicus.eu/ows?service=wms&request=GetCapabilities&amp&cql=identifier='
+    wcs_capabilities = 'emg.pass.copernicus.eu/ows?service=wcs&request=GetCapabilities&amp&cql=identifier='
+    expected_wms_link = '%s"%s"' % (wms_capabilities, product)
+    expected_wcs_link = '%s"%s"' % (wcs_capabilities, product)
+    assert expected_wms_link.replace('&amp&', '&') == wms_link
+
+    assert expected_wcs_link.replace('&amp&', '&') == wcs_link
+
+def test_reporting(sftp_connection, identifiers):
+    report_list = sftp_connection.listdir('data/to/panda/')
+    assert len(report_list) > 0
+    for item in report_list:
+        for row in identifiers:
+            identifier = row[0].split('/')[4]
+
+            if identifier in item:
+                compare_links(sftp_connection, item, identifier)
\ No newline at end of file

testing/registrar_test.sh

@@ -11,25 +11,6 @@ while read product; do
             --reporting-dir "/mnt/reports" \
             <<<$product
 
-    IFS='/' read -ra ADDR <<< "$product"
-    docker exec $(docker ps -qf "name=emg-pvs_registrar") bash -c 'ls /mnt/reports/. | grep $ADDR[4]' >> ${ADDR[4]}.csv
-
-
-    while IFS="" read -r name
-    do
-        docker exec $(docker ps -qf "name=emg-pvs_registrar") cat /mnt/reports/$name > temp.xml
-        tail -n +2 temp.xml > ${ADDR[4]}.xml
-    done < ${ADDR[4]}.csv
-
-    rm  ${ADDR[4]}.csv temp.xml
-
 done < "$product_list_file"
 
 pytest #-s registrar_test.py --name $product_list_file
-
-
-# Remove the created xml reports
-while read product; do
-    IFS='/' read -ra ADDR <<< "$product"
-    rm ${ADDR[4]}.xml
-done < $product_list_file

testing/requirements.txt

 pytest
 psycopg2
 python-dotenv
+paramiko
 # python-swiftclient
 # python-keystoneclient
\ No newline at end of file