diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 658937aad7216c60be89e3761c8e8b82192f273b..f6829ae9787c8981cb69522d05122fbc2b72616f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -82,7 +82,7 @@ build:
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME fluentd/
- docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest"
- IMAGE="$CI_REGISTRY_IMAGE/pvs_ingestor"
- - docker pull "$IMAGE":$CI_COMMIT_REF_NAME || true
+ - docker pull "$IMAGE":latest || true
- docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME ingestor/
- cd ./testing && ./gitlab_test.sh && cd -
except:
diff --git a/README.md b/README.md
index 4e74a1aab9fd822b5250f9492434ecf25c4eed57..ec20c42b13004998c58f67981bf5e5b08610c9a6 100644
--- a/README.md
+++ b/README.md
@@ -190,6 +190,16 @@ docker network create -d overlay emg-extnet
```
Add following .env files with credentials to the cloned copy of the repository /env folder: `vhr18_db.env`, `vhr18_obs.env`, `vhr18_django.env`.
+create docker secrets:
+
+Sensitive environment variables are not included in the .env files, and must be generated as docker secrets. To create docker secrets run :
+```bash
+# replace the "<variable>" with the value of the secret
+printf "<OS_PASSWORD_DOWNLOAD>" | docker secret create OS_PASSWORD_DOWNLOAD -
+printf "<DJANGO_PASSWORD>" | docker secret create DJANGO_PASSWORD -
+printf "<OS_PASSWORD>" | docker secret create OS_PASSWORD -
+```
+
Deploy the stack:
```
docker stack deploy -c docker-compose.vhr18.yml -c docker-compose.vhr18.dev.yml -c docker-compose.logging.yml -c docker-compose.logging.dev.yml vhr18-pvs # start VHR_IMAGE_2018 stack in dev mode, for example to use local sources
@@ -226,7 +236,7 @@ Select `@timestamp` as time field
The `SFTP` image allow remote access into 2 logging folders, you can define (edit/add) users, passwords and (UID/GID) in the respictive configuration file ( e.g *config/vhr_sftp_users.conf* ).
-The default username is `eox`, once the stack is deployed you can sftp into the logging folders through port 2222 on -if you rn the dev stack- localhost :
+The default username is `eox`, once the stack is deployed you can sftp into the logging folders through port 2222 on -if you are running the dev stack- localhost :
```bash
sftp -P 2222 eox@127.0.0.1
diff --git a/cache/entrypoint.sh b/cache/entrypoint.sh
index 81a77cdfc99c60a582199ac8918d02f9f5957c9b..584e50b7f247abc749e200177aff8d695cad0d5d 100644
--- a/cache/entrypoint.sh
+++ b/cache/entrypoint.sh
@@ -8,5 +8,33 @@ if [[ ! -z $SERVICES ]] ; then
wait-for-it -t $TIMEOUT $service
done
fi
+# usage: file_env VAR [DEFAULT]
+# ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+file_env() {
+ local var="$1"
+ local fileVar="${var}_FILE"
+ local def="${2:-}"
+ if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+ echo "Both $var and $fileVar are set (but are exclusive)" >&2
+ fi
+ local val="$def"
+ if [ "${!var:-}" ]; then
+ val="${!var}"
+ elif [ "${!fileVar:-}" ]; then
+ val="$(< "${!fileVar}")"
+ fi
+ cat >> /etc/bash.bashrc <<EOF
+export ${var}=${val}
+EOF
+ echo "the value of variable ${var} is set" >&2
+ # make them also available in preparatory steps until container starts
+ export ${var}=${val}
+ unset "$fileVar"
+}
+
+file_env "OS_PASSWORD"
+file_env "OS_PASSWORD_DOWNLOAD"
eval "$@"
diff --git a/cache/install.sh b/cache/install.sh
index d7c73c713b74893b196c6f9acaeedbf504fd0de6..64859f909e9e87b78c30e282597597fe9178af7f 100755
--- a/cache/install.sh
+++ b/cache/install.sh
@@ -10,7 +10,7 @@ add-apt-repository -y ppa:schpidi2/eox
apt update
echo "Installing packages"
-VERSION=1.8.0-1~bionic1eox6
+VERSION=1.10.0-1~bionic0eox2
DEBIAN_FRONTEND=noninteractive apt install -y \
libmapcache1=${VERSION} libapache2-mod-mapcache=${VERSION} mapcache-tools=${VERSION} \
sqlite3 curl apache2 python3-dateutil python3-redis wait-for-it postgresql-client
diff --git a/client/Dockerfile b/client/Dockerfile
index 2560772915d2a3ad5b8d01850c886995d1666dc7..9da01c45e88e4e9e28b8854bae6f7554d9d9bf5e 100644
--- a/client/Dockerfile
+++ b/client/Dockerfile
@@ -25,7 +25,7 @@
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
-FROM nginx
+FROM nginx:1.19
LABEL name="prism view server client" \
vendor="EOX IT Services GmbH <https://eox.at>" \
diff --git a/config/dem_mapcache-dev.xml b/config/dem_mapcache-dev.xml
index 42332ca2a01f47508c3cafd718e065f3039619b5..6a96f676635350eec2f45ba8c758d12f6c80dde2 100644
--- a/config/dem_mapcache-dev.xml
+++ b/config/dem_mapcache-dev.xml
@@ -1,4 +1,7 @@
<mapcache>
+ <connection_pool>
+ <max_connections>15</max_connections>
+ </connection_pool>
<default_format>mixed</default_format>
<format name="mypng" type="PNG">
<compression>fast</compression>
diff --git a/config/dem_mapcache-ops.xml b/config/dem_mapcache-ops.xml
index a9f2a2aa67b9a3dcf96cb48384f9c85a00314b6e..9fb0f007876fb62570d586609609935b74866043 100644
--- a/config/dem_mapcache-ops.xml
+++ b/config/dem_mapcache-ops.xml
@@ -1,4 +1,7 @@
<mapcache>
+ <connection_pool>
+ <max_connections>15</max_connections>
+ </connection_pool>
<default_format>mixed</default_format>
<format name="mypng" type="PNG">
<compression>fast</compression>
diff --git a/config/emg_mapcache-dev.xml b/config/emg_mapcache-dev.xml
index c768c922323784283a61e9111860b00c28b8f4b8..279ecd3118ef90a7e0acaacb4526d8b10a0b50cb 100644
--- a/config/emg_mapcache-dev.xml
+++ b/config/emg_mapcache-dev.xml
@@ -1,4 +1,7 @@
<mapcache>
+ <connection_pool>
+ <max_connections>15</max_connections>
+ </connection_pool>
<default_format>mixed</default_format>
<format name="mypng" type="PNG">
<compression>fast</compression>
diff --git a/config/emg_mapcache-ops.xml b/config/emg_mapcache-ops.xml
index bb4c17ab7c975b2a75de27aec62e19f4bf274f0f..02cd349edadae751c895b0e0d0fd36e848261ed4 100644
--- a/config/emg_mapcache-ops.xml
+++ b/config/emg_mapcache-ops.xml
@@ -1,4 +1,7 @@
<mapcache>
+ <connection_pool>
+ <max_connections>15</max_connections>
+ </connection_pool>
<default_format>mixed</default_format>
<format name="mypng" type="PNG">
<compression>fast</compression>
diff --git a/config/emg_preprocessor-config.yml b/config/emg_preprocessor-config.yml
index dc6fc7b08763e5069c0839ea9409f36c6336c271..13c5d5e51c5ea64d07b21b4fe6766ff312f55ac5 100644
--- a/config/emg_preprocessor-config.yml
+++ b/config/emg_preprocessor-config.yml
@@ -1,59 +1,47 @@
source:
type: swift
kwargs:
- username: !env{{OS_USERNAME_DOWNLOAD}}
- password: "!env{{OS_PASSWORD_DOWNLOAD}}"
- tenant_name: "!env{{OS_TENANT_NAME_DOWNLOAD}}"
- tenant_id: "!env{{OS_TENANT_ID_DOWNLOAD}}"
- region_name: "!env{{OS_REGION_NAME_DOWNLOAD}}"
- auth_url: "!env{{OS_AUTH_URL_DOWNLOAD}}"
- auth_version: "!env{{ST_AUTH_VERSION_DOWNLOAD}}"
- user_domain_name: !env{{OS_USER_DOMAIN_NAME_DOWNLOAD}}
-# target:
-# type: swift
-# kwargs:
-# auth_version: !env{{ST_AUTH_VERSION}}
-# auth_url: "!env{{OS_AUTH_URL}}"
-# username: "!env{{OS_USERNAME}}"
-# password: "!env{{OS_PASSWORD}}"
-# tenant_name: !env{{OS_TENANT_NAME}}
-# tenant_id: !env{{OS_TENANT_ID}}
-# region_name: !env{{OS_REGION_NAME}}
-# user_domain_name: !env{{OS_USER_DOMAIN_NAME}}
+ username: !env '${OS_USERNAME_DOWNLOAD}'
+ password: !env '${OS_PASSWORD_DOWNLOAD}'
+ tenant_name: !env '${OS_TENANT_NAME_DOWNLOAD}'
+ tenant_id: !env '${OS_TENANT_ID_DOWNLOAD}'
+ region_name: !env '${OS_REGION_NAME_DOWNLOAD}'
+ auth_url: !env '${OS_AUTH_URL_DOWNLOAD}'
+ auth_version: !env '${ST_AUTH_VERSION_DOWNLOAD}'
+ user_domain_name: !env '${OS_USER_DOMAIN_NAME_DOWNLOAD}'
target:
- type: local
- replace: true
+ type: swift
+ replace: false
kwargs:
- storage_path: /mnt/data/target
-
-workdir: /mnt/data/workdir
-keep_temp: true
-
-# metadata file to look for in downloaded tar/zip file
+ username: !env '${OS_USERNAME}'
+ password: !env '${OS_PASSWORD}'
+ tenant_name: !env '${OS_TENANT_NAME}'
+ tenant_id: !env '${OS_TENANT_ID}'
+ region_name: !env '${OS_REGION_NAME}'
+ auth_version: !env '${ST_AUTH_VERSION}'
+ auth_url: !env '${OS_AUTH_URL}'
+ user_domain_name: !env '${OS_USER_DOMAIN_NAME}'
+ container: !env '${UPLOAD_CONTAINER}'
+workdir: /tmp
+keep_temp: false
metadata_glob: "*GSC*.xml"
-
-# extractors for Product type / level
type_extractor:
- # xpath can also be a list of xpaths to be tried one after another
xpath:
- /gsc:report/gsc:opt_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
- /gsc:report/gsc:sar_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
- map: # optional mapping from extracted type name to used product type name
- PHR_FUS__3: PH00
-
level_extractor:
# xpath can also be a list of xpaths to be tried one after another
xpath: substring-after(substring-after(/gsc:report/gsc:opt_metadata/gml:metaDataProperty/gsc:EarthObservationMetaData/eop:parentIdentifier/text(), '/'), '/')
- map: # optional mapping
-
-
preprocessing:
defaults:
+ move_files: true
+ data_file_globs:
+ - '*.tif'
+ - '*.jp2'
output:
options:
- # WarpOptions (see https://gdal.org/python/osgeo.gdal-module.html#WarpOptions)
format: COG
- dstSRS: "EPSG:4326"
+ dstSRS: 'EPSG:4326'
dstNodata: 0
creationOptions:
- BLOCKSIZE=512
@@ -62,112 +50,8 @@ preprocessing:
- BIGTIFF=IF_SAFER
- OVERVIEWS=AUTO
types:
- KS03:
- nested: true
- data_file_globs:
- - "*.tif"
- additional_file_globs:
- - "*.rpc"
- georeference:
- stack_bands:
- # stack all bands for each scene in the product
- group_by: ".*/(.*)_P..tif"
- sort_by: ".*_P(R|G|B|N).tif"
- order:
- - R
- - G
- - B
- - N
-
-
-
-
- RS02: # as extracted/translated above
- # whether the package can contain sub-packages of TARs/ZIPs
- nested: true
- # glob selectors to look for source images in the source package
- data_file_globs:
- - "*.TIF"
- additional_file_globs:
- - "*.rpc"
-
- # a custom preprocessor function to be called on all selected files
- # custom_preprocessor:
- # path: "path.to.some.module:attribute"
- # # TODO: specify args/kwargs and pass meaningful parameters
-
- georeference:
- # georeference each file individually
- # - type: geotransform # one of geotransform, RPC, GCP, world file
- # - type: GCP
-
-
- stack_bands:
- # stack all bands for each scene in the product
- group_by: # TODO: figure out a way to get a grouping. e.g: part of the filename using regex?
-
- output:
-
- # define a custom postprocessor function to be called on the processed file
- # custom_postprocessor:
- # path: "path.to.some.module:attribute"
- # # TODO: specify args/kwargs and pass meaningful parameters
-
- CS00:
- nested: true
- data_file_globs:
- - "*.h5"
-
- subdatasets:
- data_file_glob: '*/*/*.h5'
- subdataset_types:
- '//S01/SBI': 'S01_SBI'
-
- georeference:
- type: corners
- corner_names: ["S01_SBI_Bottom_Left_Geodetic_Coordinates", "S01_SBI_Bottom_Right_Geodetic_Coordinates", "S01_SBI_Top_Left_Geodetic_Coordinates", "S01_SBI_Top_Right_Geodetic_Coordinates"]
- orbit_direction_name: Orbit_Direction
- force_north_up: false
- # gcp_srid:
-
- calc:
- formulas:
- - inputs:
- A:
- glob: '*.tif'
- band: 1
- B:
- glob: '*.tif'
- band: 2
- data_type: Float32
- formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
- output_postfix: _proc
- nodata_value: 0
- output:
- options:
- # WarpOptions (see https://gdal.org/python/osgeo.gdal-module.html#WarpOptions)
- format: "COG"
- dstSRS: "EPSG:3857"
- dstNodata: 0
- creationOptions:
- - BLOCKSIZE=512
- - COMPRESS=DEFLATE
- - LEVEL=6
- - OVERVIEWS=AUTO
- - NUM_THREADS=8
- - BIGTIFF=IF_SAFER
- - RESAMPLING=CUBIC
- CS01:
- nested: true
- data_file_globs:
- - "*.h5"
-
- subdatasets:
- data_file_glob: '*/*.h5'
- subdataset_types:
- '//S01/SBI': 'S01_SBI'
-
- georeference:
- type: corners
-
-# this configuration is still a stub
+ PH1B: # just to pass validation
+ nested: true
+# this configuration is still a stub - not all product types are done
+# https://gitlab.eox.at/esa/prism/vs/-/issues/56
+# https://gitlab.eox.at/esa/prism/vs/-/issues/23
diff --git a/config/vhr18_mapcache-dev.xml b/config/vhr18_mapcache-dev.xml
index 7cf52772adc81ad07ec72616f6650f95404e175c..0935921a46bd44e8a948a088b8d9d2fb0a699984 100644
--- a/config/vhr18_mapcache-dev.xml
+++ b/config/vhr18_mapcache-dev.xml
@@ -1,4 +1,7 @@
<mapcache>
+ <connection_pool>
+ <max_connections>15</max_connections>
+ </connection_pool>
<default_format>mixed</default_format>
<format name="mypng" type="PNG">
<compression>fast</compression>
diff --git a/config/vhr18_mapcache-ops.xml b/config/vhr18_mapcache-ops.xml
index b5afd384f6b2e1bb6c0e5d5eff82170266ce0a21..91ab89d0a27c0ba4a499f5bd4f1a2407c5b55c04 100644
--- a/config/vhr18_mapcache-ops.xml
+++ b/config/vhr18_mapcache-ops.xml
@@ -1,4 +1,7 @@
<mapcache>
+ <connection_pool>
+ <max_connections>15</max_connections>
+ </connection_pool>
<default_format>mixed</default_format>
<format name="mypng" type="PNG">
<compression>fast</compression>
diff --git a/core/Dockerfile b/core/Dockerfile
index 33baf310685828ebc93ededfc5b9816786491945..49ca92cfbc4d04e1b8fbf5d0bb67e78688d3a4f3 100644
--- a/core/Dockerfile
+++ b/core/Dockerfile
@@ -25,7 +25,7 @@
# IN THE SOFTWARE.
#-----------------------------------------------------------------------------
-FROM eoxa/eoxserver:latest
+FROM eoxa/eoxserver:release-1.0.0-rc12
LABEL name="prism view server core" \
vendor="EOX IT Services GmbH <https://eox.at>" \
@@ -57,9 +57,9 @@ ENV INSTANCE_ID="prism-view-server_core" \
DB_PORT= \
DB_NAME= \
INSTALL_DIR="/var/www/pvs/dev/" \
- DJANGO_USER="admin" \
- DJANGO_MAIL="office@eox.at" \
- DJANGO_PASSWORD="***REMOVED***" \
+ DJANGO_USER= \
+ DJANGO_MAIL= \
+ DJANGO_PASSWORD= \
DATA_DIR="/data/" \
APACHE_CONF="/etc/httpd/conf.d/010_pvs.conf" \
APACHE_ServerName="pvs_instance" \
diff --git a/core/entrypoint.sh b/core/entrypoint.sh
index a8e57f88358df480cba3f7e99640f455eb84f427..a3dbe58f037c4fe6301585bab2c4a7cce8dba242 100644
--- a/core/entrypoint.sh
+++ b/core/entrypoint.sh
@@ -1,4 +1,6 @@
#!/bin/bash -e
+set -eo pipefail
+shopt -s nullglob
SERVICES=${WAIT_SERVICES:=''}
TIMEOUT=${WAIT_TIMEOUT:='15'}
@@ -8,5 +10,36 @@ if [[ ! -z $SERVICES ]] ; then
wait-for-it -t $TIMEOUT $service >&2
done
fi
+# usage: file_env VAR [DEFAULT]
+# ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+file_env() {
+
+ local var="$1"
+ local fileVar="${var}_FILE"
+ local def="${2:-}"
+ if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+ echo "Both $var and $fileVar are set (but are exclusive)" >&2
+ fi
+ local val="$def"
+ if [ "${!var:-}" ]; then
+ val="${!var}"
+ elif [ "${!fileVar:-}" ]; then
+ val="$(< "${!fileVar}")"
+ fi
+ cat >> /etc/bash.bashrc <<EOF
+export ${var}=${val}
+EOF
+ echo "the value of variable ${var} is set" >&2
+ # make them also available in preparatory steps until container starts
+ export ${var}=${val}
+ unset "$fileVar"
+}
+
+file_env "OS_PASSWORD"
+file_env "OS_PASSWORD_DOWNLOAD"
+file_env "DB_PW"
+file_env "DJANGO_PASSWORD"
eval "/opt/eoxserver/eoxserver-entrypoint.sh $@"
diff --git a/docker-compose.dem.ops.yml b/docker-compose.dem.ops.yml
index b1f09109f8cead43205facab36c86a791a3a851f..6cccbfbdcba5d165994d37a0acea01ed4dec791a 100644
--- a/docker-compose.dem.ops.yml
+++ b/docker-compose.dem.ops.yml
@@ -23,13 +23,13 @@ services:
- "traefik.http.routers.dem-renderer-redirect.middlewares=redirect@file"
- "traefik.http.routers.dem-renderer-redirect.entrypoints=http"
# router for referrer based access (https)
- - "traefik.http.routers.dem-renderer_referer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.dem-renderer_referer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
- "traefik.http.routers.dem-renderer_referer.middlewares=compress@file,cors@file"
- "traefik.http.routers.dem-renderer_referer.tls=true"
- "traefik.http.routers.dem-renderer_referer.tls.certresolver=default"
- "traefik.http.routers.dem-renderer_referer.entrypoints=https"
# router for referrer based access (http)
- - "traefik.http.routers.dem-renderer_referer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.dem-renderer_referer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
- "traefik.http.routers.dem-renderer_referer-redirect.middlewares=redirect@file"
- "traefik.http.routers.dem-renderer_referer-redirect.entrypoints=http"
# general
@@ -65,13 +65,13 @@ services:
- "traefik.http.routers.dem-cache-redirect.middlewares=redirect@file"
- "traefik.http.routers.dem-cache-redirect.entrypoints=http"
# router for referrer based access (https)
- - "traefik.http.routers.dem-cache_referer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.dem-cache_referer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
- "traefik.http.routers.dem-cache_referer.middlewares=cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.dem-cache_referer.tls=true"
- "traefik.http.routers.dem-cache_referer.tls.certresolver=default"
- "traefik.http.routers.dem-cache_referer.entrypoints=https"
# router for referrer based access (http)
- - "traefik.http.routers.dem-cache_referer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.dem-cache_referer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
- "traefik.http.routers.dem-cache_referer-redirect.middlewares=redirect@file"
- "traefik.http.routers.dem-cache_referer-redirect.entrypoints=http"
# general
diff --git a/docker-compose.dem.yml b/docker-compose.dem.yml
index 611e0906f7a7b6a8ee421436af5bcd8a44574d94..13bf93d2fa973c2b229169a47f9a5ac762bfe38c 100644
--- a/docker-compose.dem.yml
+++ b/docker-compose.dem.yml
@@ -14,6 +14,11 @@ services:
constraints: [node.role == manager]
networks:
- intnet
+ command: ["postgres", "-c", "max_connections=300"]
+ sysctls:
+ net.ipv4.tcp_keepalive_time: 600
+ net.ipv4.tcp_keepalive_probes: 5
+ net.ipv4.tcp_keepalive_intvl: 10
redis:
image: redis
volumes:
@@ -36,6 +41,8 @@ services:
- env/dem_db.env
- env/dem_django.env
- env/dem_obs.env
+ secrets:
+ - DJANGO_PASSWORD
environment:
INSTANCE_ID: "prism-view-server_renderer"
INSTALL_DIR: "/var/www/pvs/dev/"
@@ -43,6 +50,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "database:5432"
+ DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
configs:
- source: init-db
target: /init-db.sh
@@ -61,11 +69,16 @@ services:
- env/dem.env
- env/dem_db.env
- env/dem_obs.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_cache"
RENDERER_HOST: renderer
WAIT_SERVICES: "database:5432 renderer:80"
WAIT_TIMEOUT: 300 # wait up to 5 minutes
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
deploy:
replicas: 1
networks:
@@ -81,10 +94,15 @@ services:
- env/dem.env
- env/dem_obs.env
- env/dem_redis.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_seeder"
RENDERER_HOST: renderer
WAIT_SERVICES: "redis:6379 database:5432"
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
deploy:
replicas: 0
networks:
@@ -97,12 +115,17 @@ services:
- env/dem.env
- env/dem_obs.env
- env/dem_redis.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_preprocessor"
WAIT_SERVICES: "redis:6379"
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
configs:
- source: preprocessor-config
- target: /config.yaml
+ target: /config.yaml
deploy:
replicas: 1
networks:
@@ -125,6 +148,9 @@ services:
- env/dem_db.env
- env/dem_obs.env
- env/dem_redis.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_registrar"
INSTALL_DIR: "/var/www/pvs/dev/"
@@ -134,6 +160,8 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "redis:6379 database:5432"
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
REPORTING_DIR: '/mnt/reports/'
configs:
- source: init-db
@@ -198,3 +226,11 @@ volumes:
report-data:
networks:
intnet:
+secrets:
+ OS_PASSWORD:
+ external: true
+ OS_PASSWORD_DOWNLOAD:
+ external: true
+ DJANGO_PASSWORD:
+ external: true
+
\ No newline at end of file
diff --git a/docker-compose.emg.ops.yml b/docker-compose.emg.ops.yml
index fcc054e6937f3a9886345f9a45b0642a0b195cea..7517f1aeb239483400662957a69a7708ccd37ab4 100644
--- a/docker-compose.emg.ops.yml
+++ b/docker-compose.emg.ops.yml
@@ -23,13 +23,13 @@ services:
- "traefik.http.routers.emg-renderer-redirect.middlewares=redirect@file"
- "traefik.http.routers.emg-renderer-redirect.entrypoints=http"
# router for referrer based access (https)
- - "traefik.http.routers.emg-renderer_referer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.emg-renderer_referer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
- "traefik.http.routers.emg-renderer_referer.middlewares=compress@file,cors@file"
- "traefik.http.routers.emg-renderer_referer.tls=true"
- "traefik.http.routers.emg-renderer_referer.tls.certresolver=default"
- "traefik.http.routers.emg-renderer_referer.entrypoints=https"
# router for referrer based access (http)
- - "traefik.http.routers.emg-renderer_referer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.emg-renderer_referer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
- "traefik.http.routers.emg-renderer_referer-redirect.middlewares=redirect@file"
- "traefik.http.routers.emg-renderer_referer-redirect.entrypoints=http"
# general
@@ -65,13 +65,13 @@ services:
- "traefik.http.routers.emg-cache-redirect.middlewares=redirect@file"
- "traefik.http.routers.emg-cache-redirect.entrypoints=http"
# router for referrer based access (https)
- - "traefik.http.routers.emg-cache_referer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.emg-cache_referer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
- "traefik.http.routers.emg-cache_referer.middlewares=cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.emg-cache_referer.tls=true"
- "traefik.http.routers.emg-cache_referer.tls.certresolver=default"
- "traefik.http.routers.emg-cache_referer.entrypoints=https"
# router for referrer based access (http)
- - "traefik.http.routers.emg-cache_referer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.emg-cache_referer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
- "traefik.http.routers.emg-cache_referer-redirect.middlewares=redirect@file"
- "traefik.http.routers.emg-cache_referer-redirect.entrypoints=http"
# general
diff --git a/docker-compose.emg.yml b/docker-compose.emg.yml
index cb226923375d692a767b13308e851143e2b43081..8903a2931a9be92e3c26ab67d4f361ab0bd31c8d 100644
--- a/docker-compose.emg.yml
+++ b/docker-compose.emg.yml
@@ -14,6 +14,11 @@ services:
constraints: [node.role == manager]
networks:
- intnet
+ command: ["postgres", "-c", "max_connections=300"]
+ sysctls:
+ net.ipv4.tcp_keepalive_time: 600
+ net.ipv4.tcp_keepalive_probes: 5
+ net.ipv4.tcp_keepalive_intvl: 10
redis:
image: redis
volumes:
@@ -36,6 +41,8 @@ services:
- env/emg_db.env
- env/emg_django.env
- env/emg_obs.env
+ secrets:
+ - DJANGO_PASSWORD
environment:
INSTANCE_ID: "prism-view-server_renderer"
INSTALL_DIR: "/var/www/pvs/dev/"
@@ -43,6 +50,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "database:5432"
+ DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
configs:
- source: init-db
target: /init-db.sh
@@ -61,11 +69,16 @@ services:
- env/emg.env
- env/emg_db.env
- env/emg_obs.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_cache"
RENDERER_HOST: renderer
WAIT_SERVICES: "database:5432 renderer:80"
WAIT_TIMEOUT: 300 # wait up to 5 minutes
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
deploy:
replicas: 1
networks:
@@ -81,10 +94,15 @@ services:
- env/emg.env
- env/emg_obs.env
- env/emg_redis.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_seeder"
RENDERER_HOST: renderer
WAIT_SERVICES: "redis:6379 database:5432"
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
deploy:
replicas: 0
networks:
@@ -107,9 +125,14 @@ services:
- env/emg.env
- env/emg_obs.env
- env/emg_redis.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_preprocessor"
WAIT_SERVICES: "redis:6379"
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
configs:
- source: preprocessor-config
target: /config.yaml
@@ -135,6 +158,9 @@ services:
- env/emg_db.env
- env/emg_obs.env
- env/emg_redis.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_registrar"
INSTALL_DIR: "/var/www/pvs/dev/"
@@ -144,6 +170,9 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "redis:6379 database:5432"
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
+
REPORTING_DIR: '/mnt/reports/'
configs:
- source: init-db
@@ -198,3 +227,10 @@ volumes:
report-data:
networks:
intnet:
+secrets:
+ OS_PASSWORD:
+ external: true
+ OS_PASSWORD_DOWNLOAD:
+ external: true
+ DJANGO_PASSWORD:
+ external: true
\ No newline at end of file
diff --git a/docker-compose.vhr18.ops.yml b/docker-compose.vhr18.ops.yml
index e8665d49ec311f51e4bc59d6550381e86f4423f4..fef2a0d2f987e35fc781a29fb376f2c78584bcd9 100644
--- a/docker-compose.vhr18.ops.yml
+++ b/docker-compose.vhr18.ops.yml
@@ -24,13 +24,13 @@ services:
- "traefik.http.routers.vhr18-renderer-redirect.middlewares=redirect@file"
- "traefik.http.routers.vhr18-renderer-redirect.entrypoints=http"
# router for referrer based access (https)
- - "traefik.http.routers.vhr18-renderer_referer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.vhr18-renderer_referer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
- "traefik.http.routers.vhr18-renderer_referer.middlewares=compress@file,cors@file"
- "traefik.http.routers.vhr18-renderer_referer.tls=true"
- "traefik.http.routers.vhr18-renderer_referer.tls.certresolver=default"
- "traefik.http.routers.vhr18-renderer_referer.entrypoints=https"
# router for referrer based access (http)
- - "traefik.http.routers.vhr18-renderer_referer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.vhr18-renderer_referer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
- "traefik.http.routers.vhr18-renderer_referer-redirect.middlewares=redirect@file"
- "traefik.http.routers.vhr18-renderer_referer-redirect.entrypoints=http"
# general
@@ -65,13 +65,13 @@ services:
- "traefik.http.routers.vhr18-cache-redirect.middlewares=redirect@file"
- "traefik.http.routers.vhr18-cache-redirect.entrypoints=http"
# router for referrer based access (https)
- - "traefik.http.routers.vhr18-cache_referer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.vhr18-cache_referer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
- "traefik.http.routers.vhr18-cache_referer.middlewares=cache-stripprefix,compress@file,cors@file"
- "traefik.http.routers.vhr18-cache_referer.tls=true"
- "traefik.http.routers.vhr18-cache_referer.tls.certresolver=default"
- "traefik.http.routers.vhr18-cache_referer.entrypoints=https"
# router for referrer based access (http)
- - "traefik.http.routers.vhr18-cache_referer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
+ - "traefik.http.routers.vhr18-cache_referer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
- "traefik.http.routers.vhr18-cache_referer-redirect.middlewares=redirect@file"
- "traefik.http.routers.vhr18-cache_referer-redirect.entrypoints=http"
# general
diff --git a/docker-compose.vhr18.yml b/docker-compose.vhr18.yml
index 3714eed57e4f8a019e6a69e32e3c9cf646300182..4d118f59d10be0dab519c32907fbfcc346804f21 100644
--- a/docker-compose.vhr18.yml
+++ b/docker-compose.vhr18.yml
@@ -14,6 +14,11 @@ services:
constraints: [node.role == manager]
networks:
- intnet
+ command: ["postgres", "-c", "max_connections=300"]
+ sysctls:
+ net.ipv4.tcp_keepalive_time: 600
+ net.ipv4.tcp_keepalive_probes: 5
+ net.ipv4.tcp_keepalive_intvl: 10
redis:
image: redis
volumes:
@@ -36,6 +41,8 @@ services:
- env/vhr18_db.env
- env/vhr18_django.env
- env/vhr18_obs.env
+ secrets:
+ - DJANGO_PASSWORD
environment:
INSTANCE_ID: "prism-view-server_renderer"
INSTALL_DIR: "/var/www/pvs/dev/"
@@ -43,6 +50,7 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "database:5432"
+ DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
configs:
- source: init-db
target: /init-db.sh
@@ -61,11 +69,16 @@ services:
- env/vhr18.env
- env/vhr18_db.env
- env/vhr18_obs.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_cache"
RENDERER_HOST: renderer
WAIT_SERVICES: "database:5432 renderer:80"
WAIT_TIMEOUT: 300 # wait up to 5 minutes
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
deploy:
replicas: 1
logging:
@@ -84,10 +97,15 @@ services:
- env/vhr18.env
- env/vhr18_obs.env
- env/vhr18_redis.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_seeder"
RENDERER_HOST: renderer
WAIT_SERVICES: "redis:6379 database:5432"
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
deploy:
replicas: 0
networks:
@@ -110,12 +128,17 @@ services:
- env/vhr18.env
- env/vhr18_obs.env
- env/vhr18_redis.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_preprocessor"
WAIT_SERVICES: "redis:6379"
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
configs:
- source: preprocessor-config
- target: /config.yaml
+ target: /config.yaml
deploy:
replicas: 1
networks:
@@ -138,6 +161,9 @@ services:
- env/vhr18_db.env
- env/vhr18_obs.env
- env/vhr18_redis.env
+ secrets:
+ - OS_PASSWORD
+ - OS_PASSWORD_DOWNLOAD
environment:
INSTANCE_ID: "prism-view-server_registrar"
INSTALL_DIR: "/var/www/pvs/dev/"
@@ -147,6 +173,8 @@ services:
INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
STARTUP_SCRIPTS: "/wait-initialized.sh"
WAIT_SERVICES: "redis:6379 database:5432"
+ OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+ OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
REPORTING_DIR: '/mnt/reports/'
configs:
- source: init-db
@@ -177,7 +205,7 @@ services:
replicas: 1
ports:
- - "2222:22"
+ - "2222:22"
ingestor:
image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
deploy:
@@ -205,3 +233,10 @@ volumes:
report-data:
networks:
intnet:
+secrets:
+ OS_PASSWORD:
+ external: true
+ OS_PASSWORD_DOWNLOAD:
+ external: true
+ DJANGO_PASSWORD:
+ external: true
diff --git a/documentation/operator-guide/setup.rst b/documentation/operator-guide/setup.rst
index fbafa38bda800a7f858170a7776c605f87862885..bb91533c06ab1e442c2ed932fd03f589be0d94d2 100644
--- a/documentation/operator-guide/setup.rst
+++ b/documentation/operator-guide/setup.rst
@@ -116,6 +116,25 @@ Now the relevant images can be pulled:
.. # TODO: ingestor image?
+
+Logging
+-------
+
+For production, the docker images in the compose files use the default logging
+driver. Therefore we configure the default logging driver for the docker daemon to
+be fluent by createing the file ``/etc/docker/daemon.json`` with the following content:
+
+.. code-block:: json
+
+ {
+ "log-driver": "fluentd"
+ }
+
+For development, we don't want to redirect all of the docker logging output,
+so the respective compose files for dev configure the logging driver for each
+container.
+
+
Stack Deployment
----------------
diff --git a/ingestor/Dockerfile b/ingestor/Dockerfile
index 6490ddff5c31c8f9eb2e9dc2bbfa452e0088434f..61afce217f6e28ee66023eadb0cd0047336f68b4 100644
--- a/ingestor/Dockerfile
+++ b/ingestor/Dockerfile
@@ -42,20 +42,10 @@ RUN mkdir /ingestor
COPY app.py config.py filedaemon.py /
COPY ingestor/ /ingestor
-ENV COLLECTION_ID= \
- INSTANCE_ID="prism-view-server_ingestor" \
- RENDERER_HOST= \
- COLLECTION= \
+ENV INSTANCE_ID="prism-view-server_ingestor" \
REDIS_HOST= \
REDIS_PORT="6379" \
REDIS_PREPROCESS_MD_QUEUE_KEY="preprocess-md_queue" \
- INOTIFY_WATCH_DIR="/mnt/data" \
- ST_AUTH_VERSION=3 \
- OS_AUTH_URL= \
- OS_USERNAME= \
- OS_PASSWORD= \
- OS_TENANT_NAME= \
- OS_TENANT_ID= \
- OS_REGION_NAME=
+ INOTIFY_WATCH_DIR="/mnt/data"
CMD ["gunicorn3", "-c", "config.py", "app"]
diff --git a/ingestor/app.py b/ingestor/app.py
index 6fde0df409198fd618ff77f3fd3752a4841bb0d3..41d50fc361d6892a885cfb5614e7ba8acaf7d838 100644
--- a/ingestor/app.py
+++ b/ingestor/app.py
@@ -83,9 +83,13 @@ def ingest():
request.get_data()
browse_report = parse_browse_report(request.data)
logger.debug(browse_report)
- client.lpush(queue_name, json.dumps(
- browse_report, default=converter
- ))
+ if isinstance(browse_report, str):
+ queue_content = browse_report
+ else:
+ queue_content = json.dumps(
+ browse_report, default=converter
+ )
+ client.lpush(queue_name, queue_content)
return Response(status=202)
except Exception as e:
diff --git a/ingestor/filedaemon.py b/ingestor/filedaemon.py
index 2b3bef423a489d6f004295382117cea3305a5fd7..cb0f0fb72fd09e24f2aec0a4bbbbc87757655f26 100644
--- a/ingestor/filedaemon.py
+++ b/ingestor/filedaemon.py
@@ -86,9 +86,13 @@ class EventHandler(pyinotify.ProcessEvent):
with open(event.pathname) as f:
browse_report = parse_browse_report(f)
logger.debug(browse_report)
- client.lpush(queue_name, json.dumps(
- browse_report, default=converter
- ))
+ if isinstance(browse_report, str):
+ queue_content = browse_report
+ else:
+ queue_content = json.dumps(
+ browse_report, default=converter
+ )
+ client.lpush(queue_name, queue_content)
except Exception as e:
logger.exception(e)
diff --git a/ingestor/ingestor/browse_report.py b/ingestor/ingestor/browse_report.py
index c3669bd77719f32609d4a8b6c344b1f30f3ed675..82f48e88f14631cbaeb7f6f5a05aa7cd029804ff 100644
--- a/ingestor/ingestor/browse_report.py
+++ b/ingestor/ingestor/browse_report.py
@@ -36,28 +36,30 @@ from .util import pairwise
class BrowseReportParserError(ValueError):
pass
+
NS_REP = 'http://ngeo.eo.esa.int/ngEO/browseReport/1.0'
NS_REP_OLD = 'http://ngeo.eo.esa.int/schema/browseReport'
NS_BSI = 'http://ngeo.eo.esa.int/schema/browse/ingestion'
-
-
-nsmap = {
- 'rep': NS_REP,
- 'bsi': NS_BSI
-}
+NS_TPZ = 'http://www.telespazio.com/CSCDA/CDD/PDAS'
def rep(tag):
return f'{{{NS_REP}}}{tag}'
+
def rep_old(tag):
return f'{{{NS_REP_OLD}}}{tag}'
+
def bsi(tag):
return f'{{{NS_BSI}}}{tag}'
-ALLOWED_ROOT_TAGS = {rep('browseReport'), rep_old('browseReport'), bsi('ingestBrowse')}
+def tpz(tag):
+ return f'{{{NS_TPZ}}}{tag}'
+
+
+ALLOWED_ROOT_TAGS = {rep('browseReport'), rep_old('browseReport'), bsi('ingestBrowse'), tpz('OnLineArchiveItem')}
def parse_browse_report(input_file):
@@ -66,7 +68,6 @@ def parse_browse_report(input_file):
"""
if isinstance(input_file, bytes):
input_file = io.BytesIO(input_file)
-
try:
tree = etree.parse(input_file)
except etree.XMLSyntaxError as e:
@@ -80,6 +81,12 @@ def parse_browse_report(input_file):
if root.tag == rep_old('browseReport'):
used_rep = rep_old
+ elif root.tag == tpz('OnLineArchiveItem'):
+ if root.findtext(tpz('status')) in ['added', 'updated']:
+ path_el = root.find(tpz('path')).findtext(tpz('URL'))
+ obs_path = path_el.replace('\n', '').replace('/', '', 1)
+ return obs_path
+ raise BrowseReportParserError('Status not in "added" or "updated", aborting.')
else:
used_rep = rep
diff --git a/ingestor/tests/data/tpzonlineitem.xml b/ingestor/tests/data/tpzonlineitem.xml
new file mode 100644
index 0000000000000000000000000000000000000000..3033c94f66cc53d736d9f9d364a310016248684b
--- /dev/null
+++ b/ingestor/tests/data/tpzonlineitem.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--Sample XML file generated by XMLSpy v2013 sp1 (http://www.altova.com)-->
+<OnLineArchiveItem xsi:schemaLocation="http://www.telespazio.com/CSCDA/CDD/PDAS PDAS_interfaces%2020190924_1916.xsd" xmlns="http://www.telespazio.com/CSCDA/CDD/PDAS" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <identifier>urn:eop:SPOT:MULTISPECTRAL_4m:DS_SPOT6_201807121048474_FR1_FR1_FR1_FR1_E007N61_04388_projected_6f78
+</identifier>
+ <path>
+ <Service>OS</Service>
+ <URL>/data25/OA/SP06/1.0/58/urn:eop:SPOT:MULTISPECTRAL_4m:DS_SPOT6_201807121048474_FR1_FR1_FR1_FR1_E007N61_04388_projected_6f78/0000/SP06_NAO_MS4_2A_20180712T104910_20180712T104914_TOU_1234_6f78.DIMA.tar
+</URL>
+ </path>
+ <status>added</status>
+</OnLineArchiveItem>
diff --git a/preprocessor/Dockerfile b/preprocessor/Dockerfile
index 1d3cb90f96f4273318e6ef21d24ab061decf677a..b7da584f7aa2e019516b9c64c87342ce2f155486 100644
--- a/preprocessor/Dockerfile
+++ b/preprocessor/Dockerfile
@@ -74,6 +74,7 @@ ENV INSTANCE_ID="prism-data-access-server_preprocessor" \
ADD run-preprocessor.sh \
entrypoint.sh \
setup.py \
+ MANIFEST.in \
/
RUN chmod +x /run-preprocessor.sh
diff --git a/preprocessor/MANIFEST.in b/preprocessor/MANIFEST.in
new file mode 100644
index 0000000000000000000000000000000000000000..276fcc8f03e1e68a3071f15d890d12bff0f0e721
--- /dev/null
+++ b/preprocessor/MANIFEST.in
@@ -0,0 +1 @@
+include preprocessor/config-schema.yaml
diff --git a/preprocessor/entrypoint.sh b/preprocessor/entrypoint.sh
index 81a77cdfc99c60a582199ac8918d02f9f5957c9b..0439d40f21c795dd4dba022b133b70dd94dc8d5a 100644
--- a/preprocessor/entrypoint.sh
+++ b/preprocessor/entrypoint.sh
@@ -8,5 +8,34 @@ if [[ ! -z $SERVICES ]] ; then
wait-for-it -t $TIMEOUT $service
done
fi
+# usage: file_env VAR [DEFAULT]
+# ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+file_env() {
+
+ local var="$1"
+ local fileVar="${var}_FILE"
+ local def="${2:-}"
+ if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+ echo "Both $var and $fileVar are set (but are exclusive)" >&2
+ fi
+ local val="$def"
+ if [ "${!var:-}" ]; then
+ val="${!var}"
+ elif [ "${!fileVar:-}" ]; then
+ val="$(< "${!fileVar}")"
+ fi
+ cat >> /etc/bash.bashrc <<EOF
+export ${var}=${val}
+EOF
+ echo "the value of variable ${var} is set" >&2
+ # make them also available in preparatory steps until container starts
+ export ${var}=${val}
+ unset "$fileVar"
+}
+
+file_env "OS_PASSWORD"
+file_env "OS_PASSWORD_DOWNLOAD"
eval "$@"
diff --git a/preprocessor/preprocessor/steps/browse_report.py b/preprocessor/preprocessor/steps/browse_report.py
index bd377b6cc3dde156079952c1288fd1e7da01df86..4e215c68e0b9ce59a7c89a12dd83ee08d3f21e37 100644
--- a/preprocessor/preprocessor/steps/browse_report.py
+++ b/preprocessor/preprocessor/steps/browse_report.py
@@ -156,7 +156,7 @@ def generate_gsc(input_filename: os.PathLike, target_filename: os.PathLike, brow
for y, x in coords
])
- return dedent(f"""\
+ xmlcontent = dedent(f"""\
<?xml version='1.0' encoding='UTF-8'?>
<gsc:report
xmlns:sar="http://earth.esa.int/sar"
@@ -208,3 +208,6 @@ def generate_gsc(input_filename: os.PathLike, target_filename: os.PathLike, brow
</gsc:opt_metadata>
</gsc:report>
""")
+ with open(target_filename, 'w') as ff:
+ ff.write(xmlcontent)
+ return xmlcontent
diff --git a/preprocessor/setup.py b/preprocessor/setup.py
index 3b90e5064267302085796c218846f79fe551945c..56f894a93464471f2c79817b005ab906dcf5bf73 100644
--- a/preprocessor/setup.py
+++ b/preprocessor/setup.py
@@ -14,6 +14,7 @@ setup(
long_description_content_type="text/markdown",
url="https://gitlab.eox.at/esa/prism/vs/-/tree/master/preprocessor",
packages=find_packages(),
+ include_package_data=True,
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License",
diff --git a/testing/envisat2.json b/testing/envisat2.json
new file mode 100644
index 0000000000000000000000000000000000000000..7c303796c546969a516f2776049cc841eb736fd3
--- /dev/null
+++ b/testing/envisat2.json
@@ -0,0 +1,18 @@
+{
+ "responsible_org_name": "SxCat",
+ "date_time": "2020-07-16T10:54:18Z",
+ "browse_type": "MER_RR__2P",
+ "browses": [{
+ "browse_identifier": "EN1_ODSI_MER_RR__2P_20120131T191319_20120131T195644_051895_0157_20180824T232647_0100",
+ "filename": "EN1_MDSI_MER_RR__2P_20080128T062653_20080128T071020_030907_0292_20180722T134645_0100.ZIP_BID.tif",
+ "image_type": "TIF",
+ "reference_system_identifier": "EPSG:4326",
+ "type": "footprint_browse",
+ "start_time": "2012-01-31T19:13:19.766Z",
+ "end_time": "2012-01-31T19:56:44.388Z",
+ "footprint": {
+ "col_row_list": [[0, 0], [0, 28], [0, 56], [0, 85], [0, 113], [0, 141], [0, 169], [0, 198], [0, 226], [0, 254], [0, 282], [0, 311], [0, 339], [0, 367], [0, 395], [0, 424], [0, 452], [0, 480], [0, 508], [0, 537], [0, 565], [0, 593], [0, 621], [0, 650], [0, 678], [0, 706], [0, 735], [0, 763], [0, 791], [0, 820], [0, 848], [0, 876], [0, 904], [0, 933], [0, 961], [0, 989], [0, 1018], [0, 1046], [0, 1074], [0, 1103], [0, 1131], [0, 1159], [0, 1188], [0, 1216], [0, 1244], [0, 1273], [0, 1301], [0, 1330], [0, 1358], [0, 1386], [0, 1415], [0, 1443], [0, 1471], [0, 1500], [0, 1528], [0, 1556], [0, 1585], [0, 1613], [0, 1642], [0, 1670], [0, 1698], [0, 1727], [0, 1755], [0, 1784], [0, 1812], [0, 1840], [0, 1869], [0, 1897], [0, 1926], [0, 1954], [0, 1983], [0, 2011], [0, 2039], [0, 2068], [0, 2096], [0, 2125], [0, 2153], [0, 2182], [0, 2210], [0, 2238], [0, 2267], [0, 2295], [0, 2324], [0, 2352], [0, 2381], [0, 2409], [0, 2438], [0, 2466], [0, 2495], [0, 2523], [0, 2551], [0, 2580], [0, 2608], [0, 2637], [0, 2665], [0, 2694], [0, 2722], [0, 2751], [0, 2779], [0, 2808], [0, 2836], [0, 2865], [0, 2893], [0, 2922], [0, 2950], [0, 2979], [0, 3007], [0, 3036], [0, 3064], [0, 3093], [0, 3121], [0, 3150], [0, 3178], [0, 3207], [0, 3235], [0, 3264], [0, 3292], [0, 3321], [0, 3349], [0, 3378], [0, 3406], [0, 3435], [0, 3463], [0, 3492], [0, 3521], [0, 3549], [0, 3578], [0, 3606], [0, 3635], [0, 3663], [0, 3692], [0, 3720], [0, 3749], [0, 3777], [0, 3806], [0, 3834], [0, 3863], [0, 3891], [0, 3920], [0, 3949], [0, 3977], [0, 4006], [0, 4034], [0, 4063], [0, 4091], [0, 4120], [0, 4148], [0, 4177], [0, 4205], [0, 4234], [0, 4263], [0, 4291], [0, 4320], [0, 4348], [0, 4377], [0, 4405], [0, 4434], [0, 4462], [0, 4491], [0, 4519], [0, 4548], [0, 4576], [0, 4605], [0, 4634], [0, 4662], [0, 4691], [0, 4719], [0, 4748], [0, 4776], [0, 4805], [0, 4833], [0, 4862], [0, 4890], [0, 4919], [0, 4948], [0, 4976], [0, 5005], [0, 5033], [0, 5062], [0, 5090], [0, 5119], [0, 5147], [0, 5176], [0, 5204], [0, 5233], [0, 5261], [0, 5290], [0, 5318], [0, 5347], [0, 5375], [0, 5404], [0, 5432], [0, 5461], [0, 5489], [0, 5518], [0, 5546], [0, 5575], [0, 5603], [0, 5632], [0, 5660], [0, 5689], [0, 5717], [0, 5746], [0, 5774], [0, 5803], [0, 5831], [0, 5860], [0, 5888], [0, 5917], [0, 5945], [0, 5973], [0, 6002], [0, 6030], [0, 6059], [0, 6087], [0, 6116], [0, 6144], [0, 6172], [0, 6201], [0, 6229], [0, 6257], [0, 6286], [0, 6314], [0, 6342], [0, 6371], [0, 6399], [0, 6427], [0, 6455], [0, 6483], [0, 6512], [0, 6540], [0, 6568], [0, 6582], [7, 6582], [14, 6582], [21, 6582], [28, 6582], [36, 6582], [43, 6582], [50, 6582], [57, 6582], [64, 6582], [71, 6582], [78, 6582], [85, 6582], [92, 6582], [100, 6582], [107, 6582], [114, 6582], [121, 6582], [128, 6582], [135, 6582], [142, 6582], [149, 6582], [157, 6582], [164, 6582], [171, 6582], [178, 6582], [185, 6582], [192, 6582], [199, 6582], [206, 6582], [213, 6582], [221, 6582], [228, 6582], [235, 6582], [242, 6582], [249, 6582], [256, 6582], [263, 6582], [270, 6582], [277, 6582], [285, 6582], [292, 6582], [299, 6582], [306, 6582], [313, 6582], [320, 6582], [327, 6582], [334, 6582], [341, 6582], [349, 6582], [356, 6582], [363, 6582], [370, 6582], [377, 6582], [384, 6582], [391, 6582], [398, 6582], [406, 6582], [413, 6582], [420, 6582], [427, 6582], [434, 6582], [441, 6582], [448, 6582], [455, 6582], [462, 6582], [470, 6582], [477, 6582], [484, 6582], [491, 6582], [498, 6582], [498, 6553], [498, 6525], [498, 6496], [498, 6468], [498, 6439], [498, 6411], [498, 6382], [498, 6354], [498, 6325], [498, 6296], [498, 6268], [498, 6239], [498, 6211], [498, 6182], [498, 6154], [498, 6125], [498, 6097], [498, 6068], [498, 6039], [498, 6011], [498, 5982], [498, 5954], [498, 5925], [498, 5897], [498, 5868], [498, 5840], [498, 5811], [498, 5783], [498, 5754], [498, 5725], [498, 5697], [498, 5668], [498, 5640], [498, 5611], [498, 5583], [498, 5554], [498, 5525], [498, 5497], [498, 5468], [498, 5440], [498, 5411], [498, 5383], [498, 5354], [498, 5326], [498, 5297], [498, 5268], [498, 5240], [498, 5211], [498, 5183], [498, 5154], [498, 5126], [498, 5097], [498, 5068], [498, 5040], [498, 5011], [498, 4983], [498, 4954], [498, 4925], [498, 4897], [498, 4868], [498, 4840], [498, 4811], [498, 4782], [498, 4754], [498, 4725], [498, 4697], [498, 4668], [498, 4640], [498, 4611], [498, 4582], [498, 4554], [498, 4525], [498, 4496], [498, 4468], [498, 4439], [498, 4411], [498, 4382], [498, 4353], [498, 4325], [498, 4296], [498, 4268], [498, 4239], [498, 4210], [498, 4182], [498, 4153], [498, 4124], [498, 4096], [498, 4067], [498, 4039], [498, 4010], [498, 3981], [498, 3953], [498, 3924], [498, 3895], [498, 3867], [498, 3838], [498, 3809], [498, 3781], [498, 3752], [498, 3723], [498, 3695], [498, 3666], [498, 3637], [498, 3609], [498, 3580], [498, 3551], [498, 3523], [498, 3494], [498, 3465], [498, 3437], [498, 3408], [498, 3379], [498, 3351], [498, 3322], [498, 3293], [498, 3265], [498, 3236], [498, 3207], [498, 3179], [498, 3150], [498, 3121], [498, 3093], [498, 3064], [498, 3035], [498, 3007], [498, 2978], [498, 2949], [498, 2921], [498, 2892], [498, 2863], [498, 2835], [498, 2806], [498, 2777], [498, 2749], [498, 2720], [498, 2691], [498, 2663], [498, 2634], [498, 2605], [498, 2577], [498, 2548], [498, 2519], [498, 2491], [498, 2462], [498, 2433], [498, 2405], [498, 2376], [498, 2347], [498, 2319], [498, 2290], [498, 2261], [498, 2233], [498, 2204], [498, 2175], [498, 2147], [498, 2118], [498, 2090], [498, 2061], [498, 2032], [498, 2004], [498, 1975], [498, 1946], [498, 1918], [498, 1889], [498, 1861], [498, 1832], [498, 1803], [498, 1775], [498, 1746], [498, 1718], [498, 1689], [498, 1661], [498, 1632], [498, 1604], [498, 1575], [498, 1547], [498, 1518], [498, 1489], [498, 1461], [498, 1433], [498, 1404], [498, 1376], [498, 1347], [498, 1319], [498, 1290], [498, 1262], [498, 1233], [498, 1205], [498, 1177], [498, 1148], [498, 1120], [498, 1092], [498, 1063], [498, 1035], [498, 1007], [498, 978], [498, 950], [498, 922], [498, 894], [498, 866], [498, 837], [498, 809], [498, 781], [498, 753], [498, 725], [498, 697], [498, 669], [498, 641], [498, 613], [498, 585], [498, 558], [498, 530], [498, 502], [498, 474], [498, 447], [498, 419], [498, 391], [498, 364], [498, 336], [498, 309], [498, 282], [498, 254], [498, 227], [498, 200], [498, 173], [498, 146], [498, 119], [498, 93], [498, 66], [498, 40], [498, 13], [498, 0], [491, 0], [484, 0], [477, 0], [470, 0], [462, 0], [455, 0], [448, 0], [441, 0], [434, 0], [427, 0], [420, 0], [413, 0], [406, 0], [398, 0], [391, 0], [384, 0], [377, 0], [370, 0], [363, 0], [356, 0], [349, 0], [341, 0], [334, 0], [327, 0], [320, 0], [313, 0], [306, 0], [299, 0], [292, 0], [285, 0], [277, 0], [270, 0], [263, 0], [256, 0], [249, 0], [242, 0], [235, 0], [228, 0], [221, 0], [213, 0], [206, 0], [199, 0], [192, 0], [185, 0], [178, 0], [171, 0], [164, 0], [157, 0], [149, 0], [142, 0], [135, 0], [128, 0], [121, 0], [114, 0], [107, 0], [100, 0], [92, 0], [85, 0], [78, 0], [71, 0], [64, 0], [57, 0], [50, 0], [43, 0], [36, 0], [28, 0], [21, 0], [14, 0], [7, 0], [0, 0]],
+ "coord_list": [[61.6082, 58.0603], [60.9506, 57.8166], [60.2926, 57.5795], [59.6344, 57.3484], [58.976, 57.123], [58.3173, 56.903], [57.6584, 56.688], [56.9992, 56.4776], [56.3399, 56.2717], [55.6803, 56.0699], [55.0205, 55.872], [54.3606, 55.6777], [53.7004, 55.487], [53.04, 55.2994], [52.3795, 55.115], [51.7188, 54.9334], [51.0579, 54.7546], [50.3969, 54.5784], [49.7356, 54.4046], [49.0742, 54.2331], [48.4127, 54.0638], [47.751, 53.8966], [47.0891, 53.7314], [46.4271, 53.568], [45.765, 53.4064], [45.1027, 53.2465], [44.4402, 53.0882], [43.7777, 52.9313], [43.1149, 52.7759], [42.4521, 52.6219], [41.7891, 52.4691], [41.126, 52.3175], [40.4628, 52.1671], [39.7994, 52.0177], [39.1359, 51.8694], [38.4723, 51.7221], [37.8086, 51.5757], [37.1447, 51.4301], [36.4808, 51.2854], [35.8167, 51.1415], [35.1525, 50.9982], [34.4883, 50.8557], [33.8239, 50.7138], [33.1594, 50.5725], [32.4948, 50.4318], [31.8301, 50.2916], [31.1653, 50.1518], [30.5005, 50.0126], [29.8355, 49.8737], [29.1705, 49.7353], [28.5053, 49.5972], [27.8401, 49.4594], [27.1748, 49.3219], [26.5095, 49.1847], [25.844, 49.0478], [25.1785, 48.911], [24.5129, 48.7744], [23.8473, 48.638], [23.1816, 48.5017], [22.5158, 48.3655], [21.85, 48.2294], [21.1841, 48.0933], [20.5182, 47.9572], [19.8522, 47.8212], [19.1862, 47.6851], [18.5202, 47.549], [17.8541, 47.4129], [17.1879, 47.2766], [16.5218, 47.1402], [15.8556, 47.0037], [15.1894, 46.867], [14.5232, 46.7302], [13.8569, 46.5931], [13.1907, 46.4558], [12.5244, 46.3183], [11.8581, 46.1805], [11.1919, 46.0424], [10.5256, 45.904], [9.85935, 45.7652], [9.19311, 45.6261], [8.52688, 45.4866], [7.86066, 45.3467], [7.19448, 45.2064], [6.52832, 45.0656], [5.86219, 44.9244], [5.1961, 44.7826], [4.53005, 44.6403], [3.86404, 44.4975], [3.19809, 44.3541], [2.53219, 44.2101], [1.86635, 44.0655], [1.20058, 43.9203], [0.534873, 43.7744], [-0.130758, 43.6278], [-0.79631, 43.4804], [-1.46178, 43.3323], [-2.12716, 43.1835], [-2.79244, 43.0338], [-3.45762, 42.8833], [-4.1227, 42.7319], [-4.78767, 42.5797], [-5.45252, 42.4265], [-6.11725, 42.2723], [-6.78185, 42.1172], [-7.44631, 41.9611], [-8.11064, 41.8039], [-8.77482, 41.6456], [-9.43885, 41.4862], [-10.1027, 41.3257], [-10.7664, 41.1639], [-11.43, 41.0009], [-12.0933, 40.8367], [-12.7565, 40.6711], [-13.4195, 40.5042], [-14.0823, 40.3359], [-14.7448, 40.1662], [-15.4072, 39.995], [-16.0693, 39.8222], [-16.7313, 39.6479], [-17.3929, 39.4719], [-18.0544, 39.2943], [-18.7156, 39.1149], [-19.3765, 38.9338], [-20.0372, 38.7508], [-20.6975, 38.5659], [-21.3576, 38.3791], [-22.0174, 38.1902], [-22.6769, 37.9992], [-23.3361, 37.8061], [-23.995, 37.6108], [-24.6535, 37.4132], [-25.3117, 37.2132], [-25.9695, 37.0108], [-26.6269, 36.8058], [-27.284, 36.5983], [-27.9407, 36.388], [-28.5969, 36.175], [-29.2528, 35.959], [-29.9082, 35.7401], [-30.5632, 35.5182], [-31.2177, 35.293], [-31.8718, 35.0646], [-32.5254, 34.8327], [-33.1784, 34.5973], [-33.831, 34.3583], [-34.483, 34.1155], [-35.1344, 33.8687], [-35.7853, 33.6179], [-36.4356, 33.3629], [-37.0852, 33.1035], [-37.7342, 32.8396], [-38.3826, 32.571], [-39.0303, 32.2975], [-39.6773, 32.019], [-40.3235, 31.7352], [-40.969, 31.4459], [-41.6138, 31.151], [-42.2576, 30.8501], [-42.9007, 30.5431], [-43.5429, 30.2297], [-44.1841, 29.9096], [-44.8244, 29.5826], [-45.4638, 29.2483], [-46.1021, 28.9066], [-46.7393, 28.5569], [-47.3754, 28.1991], [-48.0104, 27.8326], [-48.6441, 27.4573], [-49.2766, 27.0725], [-49.9078, 26.678], [-50.5375, 26.2733], [-51.1659, 25.8579], [-51.7927, 25.4312], [-52.418, 24.9929], [-53.0416, 24.5421], [-53.6634, 24.0785], [-54.2834, 23.6014], [-54.9015, 23.1099], [-55.5176, 22.6036], [-56.1316, 22.0815], [-56.7433, 21.5429], [-57.3527, 20.9869], [-57.9596, 20.4125], [-58.5639, 19.8189], [-59.1654, 19.2049], [-59.7639, 18.5694], [-60.3594, 17.9113], [-60.9515, 17.2292], [-61.5402, 16.5219], [-62.1251, 15.7878], [-62.7061, 15.0255], [-63.2829, 14.2333], [-63.8553, 13.4095], [-64.4228, 12.5522], [-64.9853, 11.6595], [-65.5424, 10.7292], [-66.0936, 9.75908], [-66.6387, 8.74687], [-67.1771, 7.69004], [-67.7083, 6.58596], [-68.2319, 5.43187], [-68.7474, 4.22488], [-69.254, 2.96198], [-69.7511, 1.64004], [-70.2381, 0.255844], [-70.7141, -1.19388], [-71.1784, -2.71244], [-71.63, -4.30314], [-72.068, -5.96921], [-72.4914, -7.71372], [-72.899, -9.53955], [-73.2899, -11.4492], [-73.6626, -13.4448], [-74.0161, -15.5279], [-74.349, -17.6991], [-74.6599, -19.9585], [-74.9474, -22.305], [-75.2103, -24.7361], [-75.4473, -27.2484], [-75.6569, -29.8367], [-75.838, -32.4944], [-75.9895, -35.2136], [-76.1105, -37.9847], [-76.2, -40.7971], [-76.2576, -43.6389], [-76.2828, -46.4973], [-76.2755, -49.3594], [-76.2356, -52.212], [-76.1635, -55.042], [-76.0596, -57.8371], [-75.9247, -60.586], [-75.7597, -63.2785], [-75.6662, -64.6008], [-75.809, -64.7747], [-75.9516, -64.9519], [-76.0942, -65.1328], [-76.2365, -65.3173], [-76.3788, -65.5056], [-76.5209, -65.6978], [-76.6628, -65.894], [-76.8046, -66.0943], [-76.9462, -66.2989], [-77.0877, -66.5079], [-77.229, -66.7215], [-77.3701, -66.9397], [-77.511, -67.1628], [-77.6518, -67.3909], [-77.7923, -67.6242], [-77.9326, -67.8628], [-78.0728, -68.107], [-78.2127, -68.3568], [-78.3524, -68.6126], [-78.4918, -68.8745], [-78.6311, -69.1427], [-78.77, -69.4175], [-78.9087, -69.6991], [-79.0472, -69.9877], [-79.1853, -70.2835], [-79.3232, -70.587], [-79.4608, -70.8982], [-79.598, -71.2176], [-79.7349, -71.5455], [-79.8715, -71.8821], [-80.0078, -72.2277], [-80.1436, -72.5829], [-80.2791, -72.9478], [-80.4142, -73.3229], [-80.5489, -73.7087], [-80.6831, -74.1054], [-80.8169, -74.5137], [-80.9502, -74.9339], [-81.0831, -75.3665], [-81.2154, -75.8121], [-81.3471, -76.2711], [-81.4783, -76.7442], [-81.6089, -77.232], [-81.7389, -77.735], [-81.8683, -78.254], [-81.9969, -78.7895], [-82.1249, -79.3424], [-82.2521, -79.9134], [-82.3785, -80.5033], [-82.504, -81.1129], [-82.6288, -81.743], [-82.7526, -82.3947], [-82.8754, -83.0688], [-82.9972, -83.7663], [-83.118, -84.4882], [-83.2377, -85.2357], [-83.3561, -86.0099], [-83.4733, -86.8119], [-83.5893, -87.643], [-83.7038, -88.5043], [-83.8168, -89.3972], [-83.9283, -90.323], [-84.0382, -91.283], [-84.1464, -92.2786], [-84.2527, -93.3113], [-84.3571, -94.3825], [-84.4595, -95.4934], [-84.5597, -96.6456], [-84.6577, -97.8405], [-84.7532, -99.0792], [-85.2478, -94.0229], [-85.6986, -87.8713], [-86.0905, -80.4067], [-86.4042, -71.4845], [-86.6178, -61.1717], [-86.7118, -49.8908], [-86.676, -38.4114], [-86.5144, -27.5847], [-86.2433, -18.0068], [-85.8841, -9.89227], [-85.4578, -3.1722], [-84.9814, 2.35138], [-84.4679, 6.90052], [-83.9265, 10.6741], [-83.3642, 13.8344], [-82.7859, 16.509], [-82.195, 18.7963], [-81.5942, 20.7721], [-80.9855, 22.4948], [-80.3704, 24.0101], [-79.75, 25.3537], [-79.1253, 26.5539], [-78.4968, 27.6332], [-77.8653, 28.6099], [-77.231, 29.4988], [-76.5945, 30.3121], [-75.956, 31.0598], [-75.3158, 31.7504], [-74.674, 32.391], [-74.0309, 32.9873], [-73.3866, 33.5446], [-72.7412, 34.0671], [-72.0948, 34.5585], [-71.4476, 35.0222], [-70.7995, 35.4608], [-70.1507, 35.8768], [-69.5013, 36.2723], [-68.8512, 36.6492], [-68.2006, 37.0093], [-67.5494, 37.3538], [-66.8977, 37.6842], [-66.2456, 38.0017], [-65.5931, 38.3071], [-64.9401, 38.6016], [-64.2868, 38.8859], [-63.6331, 39.1608], [-62.979, 39.4271], [-62.3246, 39.6852], [-61.6699, 39.9358], [-61.0149, 40.1794], [-60.3597, 40.4165], [-59.7041, 40.6475], [-59.0483, 40.8728], [-58.3922, 41.0928], [-57.7359, 41.3077], [-57.0793, 41.518], [-56.4225, 41.7239], [-55.7654, 41.9257], [-55.1082, 42.1235], [-54.4507, 42.3177], [-53.793, 42.5085], [-53.1351, 42.696], [-52.477, 42.8804], [-51.8187, 43.0619], [-51.1602, 43.2407], [-50.5015, 43.4169], [-49.8426, 43.5906], [-49.1836, 43.7621], [-48.5243, 43.9313], [-47.8649, 44.0985], [-47.2053, 44.2636], [-46.5455, 44.427], [-45.8856, 44.5885], [-45.2254, 44.7484], [-44.5652, 44.9067], [-43.9047, 45.0635], [-43.2441, 45.2188], [-42.5833, 45.3728], [-41.9224, 45.5256], [-41.2613, 45.6771], [-40.6001, 45.8274], [-39.9387, 45.9767], [-39.2772, 46.1249], [-38.6155, 46.2722], [-37.9536, 46.4186], [-37.2917, 46.564], [-36.6295, 46.7087], [-35.9673, 46.8526], [-35.3049, 46.9957], [-34.6423, 47.1382], [-33.9797, 47.28], [-33.3169, 47.4212], [-32.654, 47.5618], [-31.9909, 47.702], [-31.3277, 47.8416], [-30.6644, 47.9807], [-30.001, 48.1195], [-29.3374, 48.2578], [-28.6738, 48.3958], [-28.01, 48.5335], [-27.3461, 48.6709], [-26.6821, 48.808], [-26.018, 48.9448], [-25.3538, 49.0815], [-24.6895, 49.2179], [-24.0251, 49.3543], [-23.3606, 49.4904], [-22.696, 49.6265], [-22.0314, 49.7625], [-21.3666, 49.8985], [-20.7017, 50.0344], [-20.0368, 50.1703], [-19.3718, 50.3062], [-18.7067, 50.4422], [-18.0416, 50.5782], [-17.3763, 50.7143], [-16.711, 50.8506], [-16.0457, 50.9869], [-15.3803, 51.1235], [-14.7148, 51.2602], [-14.0493, 51.3971], [-13.3837, 51.5342], [-12.7181, 51.6716], [-12.0524, 51.8092], [-11.3867, 51.9471], [-10.721, 52.0854], [-10.0552, 52.224], [-9.38945, 52.3629], [-8.72364, 52.5022], [-8.05781, 52.642], [-7.39197, 52.7821], [-6.72613, 52.9227], [-6.06027, 53.0638], [-5.39442, 53.2054], [-4.72857, 53.3475], [-4.06273, 53.4901], [-3.3969, 53.6333], [-2.73109, 53.7771], [-2.0653, 53.9215], [-1.39954, 54.0665], [-0.733804, 54.2123], [-0.068109, 54.3587], [0.597545, 54.5058], [1.26315, 54.6537], [1.92871, 54.8023], [2.59421, 54.9518], [3.25964, 55.1021], [3.92501, 55.2532], [4.5903, 55.4052], [5.25552, 55.5582], [5.92065, 55.7121], [6.58569, 55.867], [7.25064, 56.0229], [7.91548, 56.1799], [8.58021, 56.3379], [9.24483, 56.4971], [9.90933, 56.6574], [10.5737, 56.8189], [11.2379, 56.9816], [11.902, 57.1456], [12.566, 57.3109], [13.2298, 57.4775], [13.8934, 57.6456], [14.5569, 57.815], [15.2202, 57.986], [15.8833, 58.1585], [16.5462, 58.3325], [17.2089, 58.5082], [17.8714, 58.6856], [18.5337, 58.8646], [19.1957, 59.0455], [19.8576, 59.2282], [20.5191, 59.4128], [21.1805, 59.5993], [21.8415, 59.7879], [22.5024, 59.9786], [23.1629, 60.1714], [23.8231, 60.3664], [24.483, 60.5637], [25.1427, 60.7634], [25.8019, 60.9655], [26.4609, 61.1701], [27.1195, 61.3773], [27.7778, 61.5873], [28.4357, 61.8], [29.0932, 62.0156], [29.7503, 62.2342], [30.4069, 62.4558], [31.0632, 62.6806], [31.719, 62.9087], [32.3744, 63.1402], [33.0293, 63.3753], [33.6837, 63.614], [34.3376, 63.8565], [34.9909, 64.1029], [35.6437, 64.3533], [36.296, 64.608], [36.9477, 64.867], [37.5987, 65.1305], [38.2492, 65.3988], [38.8989, 65.6719], [39.548, 65.9501], [40.1964, 66.2336], [40.8441, 66.5225], [41.491, 66.8171], [42.1371, 67.1176], [42.7824, 67.4243], [43.4269, 67.7374], [44.0704, 68.0571], [44.713, 68.3838], [45.3547, 68.7177], [45.9954, 69.0592], [46.635, 69.4086], [47.2735, 69.7661], [47.9109, 70.1323], [48.5471, 70.5074], [49.1821, 70.8918], [49.8157, 71.2861], [50.448, 71.6906], [51.0789, 72.1058], [51.7083, 72.5323], [52.3361, 72.9705], [52.9623, 73.4211], [53.5868, 73.8846], [54.2094, 74.3617], [54.8302, 74.8531], [55.4489, 75.3595], [56.0655, 75.8816], [56.68, 76.4204], [57.292, 76.9766], [57.9016, 77.5512], [58.2054, 77.8457], [58.2737, 77.5937], [58.3415, 77.3408], [58.4088, 77.0869], [58.4756, 76.832], [58.5419, 76.5762], [58.6077, 76.3194], [58.6729, 76.0616], [58.7376, 75.8029], [58.8019, 75.5432], [58.8655, 75.2826], [58.9287, 75.021], [58.9913, 74.7585], [59.0534, 74.495], [59.115, 74.2305], [59.176, 73.9652], [59.2364, 73.6988], [59.2964, 73.4316], [59.3557, 73.1634], [59.4146, 72.8942], [59.4728, 72.6242], [59.5305, 72.3532], [59.5876, 72.0812], [59.6442, 71.8084], [59.7002, 71.5346], [59.7556, 71.26], [59.8105, 70.9844], [59.8647, 70.7079], [59.9184, 70.4306], [59.9715, 70.1523], [60.024, 69.8732], [60.076, 69.5931], [60.1273, 69.3122], [60.178, 69.0304], [60.2281, 68.7478], [60.2777, 68.4643], [60.3266, 68.1799], [60.3749, 67.8947], [60.4226, 67.6087], [60.4696, 67.3218], [60.5161, 67.0341], [60.5619, 66.7456], [60.6071, 66.4562], [60.6517, 66.1661], [60.6957, 65.8752], [60.739, 65.5834], [60.7817, 65.2909], [60.8237, 64.9976], [60.8651, 64.7036], [60.9059, 64.4088], [60.946, 64.1132], [60.9854, 63.817], [61.0242, 63.5199], [61.0624, 63.2222], [61.0999, 62.9238], [61.1367, 62.6246], [61.1729, 62.3248], [61.2084, 62.0242], [61.2432, 61.7231], [61.2774, 61.4212], [61.3109, 61.1187], [61.3437, 60.8156], [61.3758, 60.5118], [61.4073, 60.2074], [61.438, 59.9024], [61.4681, 59.5968], [61.4975, 59.2906], [61.5263, 58.9839], [61.5543, 58.6766], [61.5816, 58.3687], [61.6082, 58.0603]]
+ }
+ }]
+}
\ No newline at end of file
diff --git a/testing/gitlab_test.sh b/testing/gitlab_test.sh
index 51a14fa137fa6b6a4160a91e1444fc8cef52a898..f2c9d5898f5c57eb5c4927e04053e6b21fc47587 100755
--- a/testing/gitlab_test.sh
+++ b/testing/gitlab_test.sh
@@ -23,14 +23,20 @@ set -o allexport
source ../env/emg.env
set +o allexport
+
mkdir data
docker swarm init
docker network create -d overlay emg-extnet
+
+# creat the docker secrets
+printf $OS_PASSWORD_DOWNLOAD | docker secret create OS_PASSWORD_DOWNLOAD -
+printf $DJANGO_PASSWORD | docker secret create DJANGO_PASSWORD -
+printf $OS_PASSWORD | docker secret create OS_PASSWORD -
docker stack deploy -c ../docker-compose.emg.yml -c ../docker-compose.emg.dev.yml emg-pvs
apk update && apk add bash postgresql-dev gcc python3-dev musl-dev py-pip gdal libffi-dev openssl-dev make
pip3 install -r requirements.txt
-./docker-stack-wait.sh -n renderer -n registrar -n preprocessor emg-pvs
+./docker-stack-wait.sh -n renderer -n registrar -n preprocessor -n ingestor emg-pvs
docker service ls
diff --git a/testing/testing_preprocessor_config.yml b/testing/testing_preprocessor_config.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ac160a150bfdd5ecda8f2161ab3e66947f7b4142
--- /dev/null
+++ b/testing/testing_preprocessor_config.yml
@@ -0,0 +1,131 @@
+source:
+ type: swift
+ kwargs:
+ username: !env '${OS_USERNAME}'
+ password: !env '${OS_PASSWORD}'
+ tenant_name: !env '${OS_TENANT_NAME}'
+ tenant_id: !env '${OS_TENANT_ID}'
+ region_name: !env '${OS_REGION_NAME}'
+ auth_version: !env '${ST_AUTH_VERSION}'
+ auth_url: !env '${OS_AUTH_URL}'
+ user_domain_name: !env '${OS_USER_DOMAIN_NAME}'
+ container: testing_preprocessor_stack
+target:
+ type: swift
+ replace: true
+ kwargs:
+ username: !env '${OS_USERNAME}'
+ password: !env '${OS_PASSWORD}'
+ tenant_name: !env '${OS_TENANT_NAME}'
+ tenant_id: !env '${OS_TENANT_ID}'
+ region_name: !env '${OS_REGION_NAME}'
+ auth_version: !env '${ST_AUTH_VERSION}'
+ auth_url: !env '${OS_AUTH_URL}'
+ user_domain_name: !env '${OS_USER_DOMAIN_NAME}'
+ container: preprocessor_results # change this?
+workdir: /tmp
+keep_temp: false
+metadata_glob: "*GSC*.xml"
+type_extractor:
+ xpath:
+ - /gsc:report/gsc:opt_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
+ - /gsc:report/gsc:sar_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
+level_extractor:
+ # xpath can also be a list of xpaths to be tried one after another
+ xpath: substring-after(substring-after(/gsc:report/gsc:opt_metadata/gml:metaDataProperty/gsc:EarthObservationMetaData/eop:parentIdentifier/text(), '/'), '/')
+preprocessing:
+ defaults:
+ move_files: true
+ data_file_globs:
+ - '*.tif'
+ - '*.jp2'
+ output:
+ options:
+ format: COG
+ dstSRS: 'EPSG:4326'
+ dstNodata: 0
+ creationOptions:
+ - BLOCKSIZE=512
+ - COMPRESS=DEFLATE
+ - NUM_THREADS=8
+ - BIGTIFF=IF_SAFER
+ - OVERVIEWS=AUTO
+ types:
+ KS03:
+ nested: true
+ data_file_globs:
+ - "*.tif"
+ additional_file_globs:
+ - "*.rpc*"
+ stack_bands:
+ group_by: ".*/(.*)_P..tif"
+ sort_by: ".*_P(R|G|B|N).tif"
+ order:
+ - R
+ - G
+ - B
+ - N
+ MER_RR__2P:
+ georeference:
+ type: gcp
+ tps: true
+ RS02:
+ nested: true
+ data_file_globs:
+ - "*imagery_*.tif"
+ georeference:
+ type: gcp
+ calc:
+ formulas:
+ # complex b1/b2 to b1 uint16 for all 4 polarizations
+ - inputs:
+ A:
+ glob: '*HH.tif'
+ band: 1
+ B:
+ glob: '*HH.tif'
+ band: 2
+ formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
+ output_postfix: _proc
+ nodata_value: 0
+ data_type: UInt16
+ - inputs:
+ A:
+ glob: '*HV.tif'
+ band: 1
+ B:
+ glob: '*HV.tif'
+ band: 2
+ formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
+ output_postfix: _proc
+ nodata_value: 0
+ data_type: UInt16
+ - inputs:
+ A:
+ glob: '*VH.tif'
+ band: 1
+ B:
+ glob: '*VH.tif'
+ band: 2
+ formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
+ output_postfix: _proc
+ nodata_value: 0
+ data_type: UInt16
+ - inputs:
+ A:
+ glob: '*VV.tif'
+ band: 1
+ B:
+ glob: '*VV.tif'
+ band: 2
+ formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
+ output_postfix: _proc
+ nodata_value: 0
+ data_type: UInt16
+ stack_bands:
+ sort_by: ".*_(HH|HV|VH|VV)_proc.tif"
+ order:
+ - HH
+ - HV
+ - VH
+ - VV