diff --git a/README.md b/README.md index a50a21e7080e757ec3634d84069e5dabaebaef34..4e74a1aab9fd822b5250f9492434ecf25c4eed57 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,20 @@ Select `@timestamp` as time field ([see also](https://www.elastic.co/guide/en/kibana/current/tutorial-define-index.html)). +### setup sftp + +The `SFTP` image allow remote access into 2 logging folders, you can define (edit/add) users, passwords and (UID/GID) in the respictive configuration file ( e.g *config/vhr_sftp_users.conf* ). + +The default username is `eox`, once the stack is deployed you can sftp into the logging folders through port 2222 on -if you rn the dev stack- localhost : + +```bash +sftp -P 2222 eox@127.0.0.1 +``` +You will log in into`/home/eox/data` directory which contains the 2 logging directories : `to/panda` and `from/fepd` + + **NOTE:** The mounted directory that you are directed into is *`/home/user`*, where `user` is the username, hence when changing the username in the `.conf` file, the `sftp` mounted volumes path in `docker-compse.<collection>.yml` must change respectivly. + + # Documentation ## Installation diff --git a/config/dem_sftp_users.conf b/config/dem_sftp_users.conf new file mode 100644 index 0000000000000000000000000000000000000000..a26dadbcb59ae265463143670502c500c72f06c1 --- /dev/null +++ b/config/dem_sftp_users.conf @@ -0,0 +1 @@ +***REMOVED***:1001:100 \ No newline at end of file diff --git a/config/emg_sftp_users.conf b/config/emg_sftp_users.conf new file mode 100644 index 0000000000000000000000000000000000000000..a26dadbcb59ae265463143670502c500c72f06c1 --- /dev/null +++ b/config/emg_sftp_users.conf @@ -0,0 +1 @@ +***REMOVED***:1001:100 \ No newline at end of file diff --git a/config/vhr18_sftp_users.conf b/config/vhr18_sftp_users.conf new file mode 100644 index 0000000000000000000000000000000000000000..a26dadbcb59ae265463143670502c500c72f06c1 --- /dev/null +++ b/config/vhr18_sftp_users.conf @@ -0,0 +1 @@ +***REMOVED***:1001:100 \ No newline at end of file diff --git a/docker-compose.dem.yml b/docker-compose.dem.yml index 2016ed58c5a7856819b8c125d7b245b8f789721a..611e0906f7a7b6a8ee421436af5bcd8a44574d94 100644 --- a/docker-compose.dem.yml +++ b/docker-compose.dem.yml @@ -148,6 +148,23 @@ services: image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest deploy: replicas: 1 + sftp: + image: atmoz/sftp:latest + volumes: + - type: volume + source: report-data + target: /home/eox/data/to/panda + - type: volume + source: from-fepd + target: /home/eox/data/from/fepd + configs: + - source: sftp-users + target: /etc/sftp/users.conf + + ports: + - "2222:22" + deploy: + replicas: 1 ingestor: image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest env_file: @@ -159,6 +176,8 @@ services: networks: - intnet configs: + sftp-users: + file: ./config/dem_sftp_users.conf init-db: file: ./config/dem_init-db.sh mapcache-dev: @@ -175,6 +194,7 @@ volumes: db-data: redis-data: instance-data: + from-fepd: report-data: networks: intnet: diff --git a/docker-compose.emg.yml b/docker-compose.emg.yml index 1b1c7be8494a19161dfcc57113cebddc05e3d0ee..cb226923375d692a767b13308e851143e2b43081 100644 --- a/docker-compose.emg.yml +++ b/docker-compose.emg.yml @@ -158,7 +158,26 @@ services: image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest deploy: replicas: 1 + sftp: + image: atmoz/sftp:latest + volumes: + - type: volume + source: report-data + target: /home/eox/data/to/panda + - type: volume + source: from-fepd + target: /home/eox/data/from/fepd + configs: + - source: sftp-users + target: /etc/sftp/users.conf + + ports: + - "2222:22" + deploy: + replicas: 1 configs: + sftp-users: + file: ./config/emg_sftp_users.conf init-db: file: ./config/emg_init-db.sh mapcache-dev: @@ -175,6 +194,7 @@ volumes: db-data: redis-data: instance-data: + from-fepd: report-data: networks: intnet: diff --git a/docker-compose.vhr18.yml b/docker-compose.vhr18.yml index e567e41ab0ad8dfb54ef30be8124bbf4f78cfa90..3714eed57e4f8a019e6a69e32e3c9cf646300182 100644 --- a/docker-compose.vhr18.yml +++ b/docker-compose.vhr18.yml @@ -161,7 +161,30 @@ services: image: registry.gitlab.eox.at/esa/prism/vs/pvs_client:latest deploy: replicas: 1 + sftp: + image: atmoz/sftp:latest + volumes: + - type: volume + source: report-data + target: /home/eox/data/to/panda + - type: volume + source: from-fepd + target: /home/eox/data/from/fepd + configs: + - source: sftp-users + target: /etc/sftp/users.conf + deploy: + replicas: 1 + + ports: + - "2222:22" + ingestor: + image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest + deploy: + replicas: 1 configs: + sftp-users: + file: ./config/vhr18_sftp_users.conf init-db: file: ./config/vhr18_init-db.sh mapcache-dev: @@ -178,6 +201,7 @@ volumes: db-data: redis-data: instance-data: + from-fepd: report-data: networks: intnet: diff --git a/testing/gitlab_test.sh b/testing/gitlab_test.sh index a4252cb4a9fdb208045cd4e92e9fb979f478f891..51a14fa137fa6b6a4160a91e1444fc8cef52a898 100755 --- a/testing/gitlab_test.sh +++ b/testing/gitlab_test.sh @@ -27,7 +27,7 @@ mkdir data docker swarm init docker network create -d overlay emg-extnet docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.yml emg-pvs -apk update && apk add bash postgresql-dev gcc python3-dev musl-dev py-pip gdal +apk update && apk add bash postgresql-dev gcc python3-dev musl-dev py-pip gdal libffi-dev openssl-dev make pip3 install -r requirements.txt ./docker-stack-wait.sh -n renderer -n registrar -n preprocessor emg-pvs diff --git a/testing/registrar_test.py b/testing/registrar_test.py index 48436d4ea82b26adee54de22afd6580ca2a51bb5..54bc3108fc0164c0e62b700c2e62b887650f7435 100644 --- a/testing/registrar_test.py +++ b/testing/registrar_test.py @@ -3,7 +3,9 @@ import csv import pytest import psycopg2 +import paramiko from dotenv import load_dotenv +from xml.etree import ElementTree @pytest.fixture(scope="session") @@ -28,6 +30,14 @@ def identifiers(): yield csv.reader(f) +@pytest.fixture +def sftp_connection(): + + transport = paramiko.Transport(('docker',2222)) + transport.connect(username='eox', password='password') + with paramiko.SFTPClient.from_transport(transport) as sftp: + yield sftp + def query_eo_object(connection, eo_id): query = f"SELECT identifier FROM coverages_eoobject WHERE identifier = '{eo_id}';" with connection.cursor() as cursor: @@ -39,3 +49,34 @@ def test_db_name(connection, identifiers): for row in identifiers: identifier = row[0].split('/')[4] query_eo_object(connection, identifier) + + +def compare_links(sftp, product_xml, product): + + report= sftp.file("data/to/panda/%s" % product_xml) + + + xml_file= report.read() + root = ElementTree.fromstring(xml_file.decode('utf-8').strip()) + + urls = root.findall('{http://www.telespazio.com/CSCDA/CDD/PDAS}URL') + wms_link = urls[0].find('{http://www.telespazio.com/CSCDA/CDD/PDAS}URL').text + wcs_link = urls[1].find('{http://www.telespazio.com/CSCDA/CDD/PDAS}URL').text + + wms_capabilities = 'emg.pass.copernicus.eu/ows?service=wms&request=GetCapabilities&&cql=identifier=' + wcs_capabilities = 'emg.pass.copernicus.eu/ows?service=wcs&request=GetCapabilities&&cql=identifier=' + expected_wms_link = '%s"%s"' % (wms_capabilities, product) + expected_wcs_link = '%s"%s"' % (wcs_capabilities, product) + assert expected_wms_link.replace('&&', '&') == wms_link + + assert expected_wcs_link.replace('&&', '&') == wcs_link + +def test_reporting(sftp_connection, identifiers): + report_list = sftp_connection.listdir('data/to/panda/') + assert len(report_list) > 0 + for item in report_list: + for row in identifiers: + identifier = row[0].split('/')[4] + + if identifier in item: + compare_links(sftp_connection, item, identifier) \ No newline at end of file diff --git a/testing/registrar_test.sh b/testing/registrar_test.sh index 536d113dfcab5446ab77c3f27b4d480922aca55f..3c26bc87dea352ee81cea9b046f0ff2602de11f6 100755 --- a/testing/registrar_test.sh +++ b/testing/registrar_test.sh @@ -11,25 +11,6 @@ while read product; do --reporting-dir "/mnt/reports" \ <<<$product - IFS='/' read -ra ADDR <<< "$product" - docker exec $(docker ps -qf "name=emg-pvs_registrar") bash -c 'ls /mnt/reports/. | grep $ADDR[4]' >> ${ADDR[4]}.csv - - - while IFS="" read -r name - do - docker exec $(docker ps -qf "name=emg-pvs_registrar") cat /mnt/reports/$name > temp.xml - tail -n +2 temp.xml > ${ADDR[4]}.xml - done < ${ADDR[4]}.csv - - rm ${ADDR[4]}.csv temp.xml - done < "$product_list_file" pytest #-s registrar_test.py --name $product_list_file - - -# Remove the created xml reports -while read product; do - IFS='/' read -ra ADDR <<< "$product" - rm ${ADDR[4]}.xml -done < $product_list_file diff --git a/testing/requirements.txt b/testing/requirements.txt index 43d82cd782b2b2d1884b394390eaf0abab8aaeac..b6db4b77a42837295c7f2c82dfad95859ea34b56 100644 --- a/testing/requirements.txt +++ b/testing/requirements.txt @@ -1,5 +1,6 @@ pytest psycopg2 python-dotenv +paramiko # python-swiftclient # python-keystoneclient \ No newline at end of file