Merge branch 'master' into chart

.gitlab-ci.yml

@@ -82,7 +82,7 @@ build:
     - docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME fluentd/
     - docker tag "$IMAGE:$CI_COMMIT_REF_NAME" "$IMAGE:latest"
     - IMAGE="$CI_REGISTRY_IMAGE/pvs_ingestor"
-    - docker pull "$IMAGE":$CI_COMMIT_REF_NAME || true
+    - docker pull "$IMAGE":latest || true
     - docker build --cache-from "$IMAGE":latest -t "$IMAGE":$CI_COMMIT_REF_SLUG -t "$IMAGE":$CI_COMMIT_REF_NAME ingestor/
     - cd ./testing && ./gitlab_test.sh && cd -
   except:

README.md

@@ -190,6 +190,16 @@ docker network create -d overlay emg-extnet
 ```
 Add following .env files with credentials to the cloned copy of the repository /env folder: `vhr18_db.env`, `vhr18_obs.env`, `vhr18_django.env`.
 
+create docker secrets:
+
+Sensitive environment variables are not included in the .env files, and must be generated as docker secrets. To create docker secrets run :
+```bash
+# replace the "<variable>" with the value of the secret
+printf "<OS_PASSWORD_DOWNLOAD>" | docker secret create OS_PASSWORD_DOWNLOAD -
+printf "<DJANGO_PASSWORD>" | docker secret create DJANGO_PASSWORD -
+printf "<OS_PASSWORD>" | docker secret create OS_PASSWORD -
+```
+
 Deploy the stack:
 ```
 docker stack deploy -c docker-compose.vhr18.yml -c docker-compose.vhr18.dev.yml -c docker-compose.logging.yml -c docker-compose.logging.dev.yml vhr18-pvs  # start VHR_IMAGE_2018 stack in dev mode, for example to use local sources
@@ -226,7 +236,7 @@ Select `@timestamp` as time field
 
 The `SFTP` image allow remote access into 2 logging folders, you can define (edit/add) users, passwords and (UID/GID) in the respictive configuration file ( e.g  *config/vhr_sftp_users.conf* ). 
 
-The default username is `eox`, once the stack is deployed you can sftp into the logging folders through port 2222 on -if you rn the dev stack- localhost :
+The default username is `eox`, once the stack is deployed you can sftp into the logging folders through port 2222 on -if you are running the dev stack- localhost :
 
 ```bash
 sftp -P 2222 eox@127.0.0.1

cache/entrypoint.sh

@@ -8,5 +8,33 @@ if [[ ! -z $SERVICES ]] ; then
         wait-for-it -t $TIMEOUT $service
     done
 fi
+# usage: file_env VAR [DEFAULT]
+#    ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+#  "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+file_env() {
 
+	local var="$1"
+	local fileVar="${var}_FILE"
+	local def="${2:-}"
+	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+		echo "Both $var and $fileVar are set (but are exclusive)" >&2
+	fi
+	local val="$def"
+	if [ "${!var:-}" ]; then
+		val="${!var}"
+	elif [ "${!fileVar:-}" ]; then
+		val="$(< "${!fileVar}")"
+	fi
+	cat >> /etc/bash.bashrc <<EOF
+export ${var}=${val}
+EOF
+	echo "the value of variable ${var} is set" >&2
+  # make them also available in preparatory steps until container starts
+  export ${var}=${val}
+	unset "$fileVar"
+}
+
+file_env "OS_PASSWORD"
+file_env "OS_PASSWORD_DOWNLOAD"
 eval "$@"

cache/install.sh

@@ -10,7 +10,7 @@ add-apt-repository -y ppa:schpidi2/eox
 apt update
 
 echo "Installing packages"
-VERSION=1.8.0-1~bionic1eox6
+VERSION=1.10.0-1~bionic0eox2
 DEBIAN_FRONTEND=noninteractive apt install -y \
     libmapcache1=${VERSION} libapache2-mod-mapcache=${VERSION} mapcache-tools=${VERSION} \
     sqlite3 curl apache2 python3-dateutil python3-redis wait-for-it postgresql-client

client/Dockerfile

@@ -25,7 +25,7 @@
 # IN THE SOFTWARE.
 #-----------------------------------------------------------------------------
 
-FROM nginx
+FROM nginx:1.19
 
 LABEL name="prism view server client" \
       vendor="EOX IT Services GmbH <https://eox.at>" \

config/dem_mapcache-dev.xml

 <mapcache>
+  <connection_pool>
+    <max_connections>15</max_connections>
+  </connection_pool>
   <default_format>mixed</default_format>
   <format name="mypng" type="PNG">
     <compression>fast</compression>

config/dem_mapcache-ops.xml

 <mapcache>
+  <connection_pool>
+    <max_connections>15</max_connections>
+  </connection_pool>
   <default_format>mixed</default_format>
   <format name="mypng" type="PNG">
     <compression>fast</compression>

config/emg_mapcache-dev.xml

 <mapcache>
+  <connection_pool>
+    <max_connections>15</max_connections>
+  </connection_pool>
   <default_format>mixed</default_format>
   <format name="mypng" type="PNG">
     <compression>fast</compression>

config/emg_mapcache-ops.xml

 <mapcache>
+  <connection_pool>
+    <max_connections>15</max_connections>
+  </connection_pool>
   <default_format>mixed</default_format>
   <format name="mypng" type="PNG">
     <compression>fast</compression>

config/emg_preprocessor-config.yml

 source:
   type: swift
   kwargs:
-    username: !env{{OS_USERNAME_DOWNLOAD}}
-    password: "!env{{OS_PASSWORD_DOWNLOAD}}"
-    tenant_name: "!env{{OS_TENANT_NAME_DOWNLOAD}}"
-    tenant_id: "!env{{OS_TENANT_ID_DOWNLOAD}}"
-    region_name: "!env{{OS_REGION_NAME_DOWNLOAD}}"
-    auth_url: "!env{{OS_AUTH_URL_DOWNLOAD}}"
-    auth_version: "!env{{ST_AUTH_VERSION_DOWNLOAD}}"
-    user_domain_name: !env{{OS_USER_DOMAIN_NAME_DOWNLOAD}}
-# target:
-#   type: swift
-#   kwargs:
-#     auth_version: !env{{ST_AUTH_VERSION}}
-#     auth_url: "!env{{OS_AUTH_URL}}"
-#     username: "!env{{OS_USERNAME}}"
-#     password: "!env{{OS_PASSWORD}}"
-#     tenant_name: !env{{OS_TENANT_NAME}}
-#     tenant_id: !env{{OS_TENANT_ID}}
-#     region_name: !env{{OS_REGION_NAME}}
-#     user_domain_name: !env{{OS_USER_DOMAIN_NAME}}
+    username: !env '${OS_USERNAME_DOWNLOAD}'
+    password: !env '${OS_PASSWORD_DOWNLOAD}'
+    tenant_name: !env '${OS_TENANT_NAME_DOWNLOAD}'
+    tenant_id: !env '${OS_TENANT_ID_DOWNLOAD}'
+    region_name: !env '${OS_REGION_NAME_DOWNLOAD}'
+    auth_url: !env '${OS_AUTH_URL_DOWNLOAD}'
+    auth_version: !env '${ST_AUTH_VERSION_DOWNLOAD}'
+    user_domain_name: !env '${OS_USER_DOMAIN_NAME_DOWNLOAD}'
 target:
-  type: local
-  replace: true
+  type: swift
+  replace: false
   kwargs:
-    storage_path: /mnt/data/target
-
-workdir: /mnt/data/workdir
-keep_temp: true
-
-# metadata file to look for in downloaded tar/zip file
+    username: !env '${OS_USERNAME}'
+    password: !env '${OS_PASSWORD}'
+    tenant_name: !env '${OS_TENANT_NAME}'
+    tenant_id: !env '${OS_TENANT_ID}'
+    region_name: !env '${OS_REGION_NAME}'
+    auth_version: !env '${ST_AUTH_VERSION}'
+    auth_url: !env '${OS_AUTH_URL}'
+    user_domain_name: !env '${OS_USER_DOMAIN_NAME}'
+    container: !env '${UPLOAD_CONTAINER}'
+workdir: /tmp
+keep_temp: false
 metadata_glob: "*GSC*.xml"
-
-# extractors for Product type / level
 type_extractor:
-  # xpath can also be a list of xpaths to be tried one after another
   xpath:
     - /gsc:report/gsc:opt_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
     - /gsc:report/gsc:sar_metadata/gml:using/eop:EarthObservationEquipment/eop:platform/eop:Platform/eop:shortName/text()
-  map: # optional mapping from extracted type name to used product type name
-    PHR_FUS__3: PH00
-
 level_extractor:
   # xpath can also be a list of xpaths to be tried one after another
   xpath: substring-after(substring-after(/gsc:report/gsc:opt_metadata/gml:metaDataProperty/gsc:EarthObservationMetaData/eop:parentIdentifier/text(), '/'), '/')
-  map: # optional mapping
-
-
 preprocessing:
   defaults:
+    move_files: true
+    data_file_globs:
+      - '*.tif'
+      - '*.jp2'
     output:
       options:
-        # WarpOptions (see https://gdal.org/python/osgeo.gdal-module.html#WarpOptions)
         format: COG
-        dstSRS: "EPSG:4326"
+        dstSRS: 'EPSG:4326'
         dstNodata: 0
         creationOptions:
           - BLOCKSIZE=512
@@ -62,112 +50,8 @@ preprocessing:
           - BIGTIFF=IF_SAFER
           - OVERVIEWS=AUTO
   types:
-    KS03:
-      nested: true
-      data_file_globs:
-        - "*.tif"
-      additional_file_globs:
-        - "*.rpc"
-      georeference:
-      stack_bands:
-        # stack all bands for each scene in the product
-        group_by: ".*/(.*)_P..tif"
-        sort_by: ".*_P(R|G|B|N).tif"
-        order:
-          - R
-          - G
-          - B
-          - N
-
-
-
-
-    RS02: # as extracted/translated above
-      # whether the package can contain sub-packages of TARs/ZIPs
-      nested: true
-      # glob selectors to look for source images in the source package
-      data_file_globs:
-        - "*.TIF"
-      additional_file_globs:
-        - "*.rpc"
-
-      # a custom preprocessor function to be called on all selected files
-      # custom_preprocessor:
-      #   path: "path.to.some.module:attribute"
-      #   # TODO: specify args/kwargs and pass meaningful parameters
-
-      georeference:
-        # georeference each file individually
-        # - type: geotransform # one of geotransform, RPC, GCP, world file
-        # - type: GCP
-
-
-      stack_bands:
-        # stack all bands for each scene in the product
-        group_by: # TODO: figure out a way to get a grouping. e.g: part of the filename using regex?
-
-      output:
-
-      # define a custom postprocessor function to be called on the processed file
-      # custom_postprocessor:
-      #   path: "path.to.some.module:attribute"
-      #   # TODO: specify args/kwargs and pass meaningful parameters
-
-    CS00:
-      nested: true
-      data_file_globs:
-        - "*.h5"
-
-      subdatasets:
-        data_file_glob: '*/*/*.h5'
-        subdataset_types:
-          '//S01/SBI': 'S01_SBI'
-
-      georeference:
-        type: corners
-        corner_names: ["S01_SBI_Bottom_Left_Geodetic_Coordinates", "S01_SBI_Bottom_Right_Geodetic_Coordinates", "S01_SBI_Top_Left_Geodetic_Coordinates", "S01_SBI_Top_Right_Geodetic_Coordinates"]
-        orbit_direction_name: Orbit_Direction
-        force_north_up: false
-        # gcp_srid:
-
-      calc:
-        formulas:
-          - inputs:
-              A:
-                glob: '*.tif'
-                band: 1
-              B:
-                glob: '*.tif'
-                band: 2
-            data_type: Float32
-            formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
-            output_postfix: _proc
-            nodata_value: 0
-      output:
-        options:
-          # WarpOptions (see https://gdal.org/python/osgeo.gdal-module.html#WarpOptions)
-          format: "COG"
-          dstSRS: "EPSG:3857"
-          dstNodata: 0
-          creationOptions:
-            - BLOCKSIZE=512
-            - COMPRESS=DEFLATE
-            - LEVEL=6
-            - OVERVIEWS=AUTO
-            - NUM_THREADS=8
-            - BIGTIFF=IF_SAFER
-            - RESAMPLING=CUBIC
-    CS01:
-      nested: true
-      data_file_globs:
-        - "*.h5"
-
-      subdatasets:
-        data_file_glob: '*/*.h5'
-        subdataset_types:
-          '//S01/SBI': 'S01_SBI'
-
-      georeference:
-        type: corners
-
-# this configuration is still a stub
+    PH1B: # just to pass validation
+      nested: true 
+# this configuration is still a stub - not all product types are done 
+# https://gitlab.eox.at/esa/prism/vs/-/issues/56
+# https://gitlab.eox.at/esa/prism/vs/-/issues/23

config/vhr18_mapcache-dev.xml

 <mapcache>
+  <connection_pool>
+    <max_connections>15</max_connections>
+  </connection_pool>
   <default_format>mixed</default_format>
   <format name="mypng" type="PNG">
     <compression>fast</compression>

config/vhr18_mapcache-ops.xml

 <mapcache>
+  <connection_pool>
+    <max_connections>15</max_connections>
+  </connection_pool>
   <default_format>mixed</default_format>
   <format name="mypng" type="PNG">
     <compression>fast</compression>

core/Dockerfile

@@ -25,7 +25,7 @@
 # IN THE SOFTWARE.
 #-----------------------------------------------------------------------------
 
-FROM eoxa/eoxserver:latest
+FROM eoxa/eoxserver:release-1.0.0-rc12
 
 LABEL name="prism view server core" \
       vendor="EOX IT Services GmbH <https://eox.at>" \
@@ -57,9 +57,9 @@ ENV INSTANCE_ID="prism-view-server_core" \
     DB_PORT= \
     DB_NAME= \
     INSTALL_DIR="/var/www/pvs/dev/" \
-    DJANGO_USER="admin" \
-    DJANGO_MAIL="office@eox.at" \
-    DJANGO_PASSWORD="***REMOVED***" \
+    DJANGO_USER= \
+    DJANGO_MAIL= \
+    DJANGO_PASSWORD= \
     DATA_DIR="/data/" \
     APACHE_CONF="/etc/httpd/conf.d/010_pvs.conf" \
     APACHE_ServerName="pvs_instance" \

core/entrypoint.sh

 #!/bin/bash -e
+set -eo pipefail
+shopt -s nullglob
 
 SERVICES=${WAIT_SERVICES:=''}
 TIMEOUT=${WAIT_TIMEOUT:='15'}
@@ -8,5 +10,36 @@ if [[ ! -z $SERVICES ]] ; then
         wait-for-it -t $TIMEOUT $service >&2
     done
 fi
+# usage: file_env VAR [DEFAULT]
+#    ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+#  "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+file_env() {
+
+	local var="$1"
+	local fileVar="${var}_FILE"
+	local def="${2:-}"
+	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+		echo "Both $var and $fileVar are set (but are exclusive)" >&2
+	fi
+	local val="$def"
+	if [ "${!var:-}" ]; then
+		val="${!var}"
+	elif [ "${!fileVar:-}" ]; then
+		val="$(< "${!fileVar}")"
+	fi
+	cat >> /etc/bash.bashrc <<EOF
+export ${var}=${val}
+EOF
+	echo "the value of variable ${var} is set" >&2
+  # make them also available in preparatory steps until container starts
+  export ${var}=${val}
+	unset "$fileVar"
+}
+
+file_env "OS_PASSWORD"
+file_env "OS_PASSWORD_DOWNLOAD"
+file_env "DB_PW"
+file_env "DJANGO_PASSWORD"
 
 eval "/opt/eoxserver/eoxserver-entrypoint.sh $@"

docker-compose.dem.ops.yml

@@ -23,13 +23,13 @@ services:
         - "traefik.http.routers.dem-renderer-redirect.middlewares=redirect@file"
         - "traefik.http.routers.dem-renderer-redirect.entrypoints=http"
         # router for referrer based access (https)
-        - "traefik.http.routers.dem-renderer_referer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.dem-renderer_referer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.dem-renderer_referer.middlewares=compress@file,cors@file"
         - "traefik.http.routers.dem-renderer_referer.tls=true"
         - "traefik.http.routers.dem-renderer_referer.tls.certresolver=default"
         - "traefik.http.routers.dem-renderer_referer.entrypoints=https"
         # router for referrer based access (http)
-        - "traefik.http.routers.dem-renderer_referer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.dem-renderer_referer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.dem-renderer_referer-redirect.middlewares=redirect@file"
         - "traefik.http.routers.dem-renderer_referer-redirect.entrypoints=http"
         # general
@@ -65,13 +65,13 @@ services:
         - "traefik.http.routers.dem-cache-redirect.middlewares=redirect@file"
         - "traefik.http.routers.dem-cache-redirect.entrypoints=http"
         # router for referrer based access (https)
-        - "traefik.http.routers.dem-cache_referer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.dem-cache_referer.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.dem-cache_referer.middlewares=cache-stripprefix,compress@file,cors@file"
         - "traefik.http.routers.dem-cache_referer.tls=true"
         - "traefik.http.routers.dem-cache_referer.tls.certresolver=default"
         - "traefik.http.routers.dem-cache_referer.entrypoints=https"
         # router for referrer based access (http)
-        - "traefik.http.routers.dem-cache_referer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.dem-cache_referer-redirect.rule=Host(`dem.pdas.prism.eox.at`, `a.dem.pdas.prism.eox.at`, `b.dem.pdas.prism.eox.at`, `c.dem.pdas.prism.eox.at`, `d.dem.pdas.prism.eox.at`, `e.dem.pdas.prism.eox.at`, `f.dem.pdas.prism.eox.at`, `g.dem.pdas.prism.eox.at`, `h.dem.pdas.prism.eox.at`, `dem.pass.copernicus.eu`, `a.dem.pass.copernicus.eu`, `b.dem.pass.copernicus.eu`, `c.dem.pass.copernicus.eu`, `d.dem.pass.copernicus.eu`, `e.dem.pass.copernicus.eu`, `f.dem.pass.copernicus.eu`, `g.dem.pass.copernicus.eu`, `h.dem.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|dem.pdas.prism.eox.at|dem.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.dem-cache_referer-redirect.middlewares=redirect@file"
         - "traefik.http.routers.dem-cache_referer-redirect.entrypoints=http"
         # general

docker-compose.dem.yml

@@ -14,6 +14,11 @@ services:
         constraints: [node.role == manager]
     networks:
       - intnet
+    command: ["postgres", "-c", "max_connections=300"]
+    sysctls:
+      net.ipv4.tcp_keepalive_time: 600
+      net.ipv4.tcp_keepalive_probes: 5
+      net.ipv4.tcp_keepalive_intvl: 10
   redis:
     image: redis
     volumes:
@@ -36,6 +41,8 @@ services:
       - env/dem_db.env
       - env/dem_django.env
       - env/dem_obs.env
+    secrets:
+      - DJANGO_PASSWORD
     environment:
       INSTANCE_ID: "prism-view-server_renderer"
       INSTALL_DIR: "/var/www/pvs/dev/"
@@ -43,6 +50,7 @@ services:
       INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
       STARTUP_SCRIPTS: "/wait-initialized.sh"
       WAIT_SERVICES: "database:5432"
+      DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
     configs:
       - source: init-db
         target: /init-db.sh
@@ -61,11 +69,16 @@ services:
       - env/dem.env
       - env/dem_db.env
       - env/dem_obs.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_cache"
       RENDERER_HOST: renderer
       WAIT_SERVICES: "database:5432 renderer:80"
       WAIT_TIMEOUT: 300  # wait up to 5 minutes
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
     deploy:
       replicas: 1
     networks:
@@ -81,10 +94,15 @@ services:
       - env/dem.env
       - env/dem_obs.env
       - env/dem_redis.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_seeder"
       RENDERER_HOST: renderer
       WAIT_SERVICES: "redis:6379 database:5432"
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
     deploy:
       replicas: 0
     networks:
@@ -97,12 +115,17 @@ services:
       - env/dem.env
       - env/dem_obs.env
       - env/dem_redis.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_preprocessor"
       WAIT_SERVICES: "redis:6379"
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
     configs:
       - source: preprocessor-config
-        target: /config.yaml
+        target: /config.yaml      
     deploy:
       replicas: 1
     networks:
@@ -125,6 +148,9 @@ services:
       - env/dem_db.env
       - env/dem_obs.env
       - env/dem_redis.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_registrar"
       INSTALL_DIR: "/var/www/pvs/dev/"
@@ -134,6 +160,8 @@ services:
       INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
       STARTUP_SCRIPTS: "/wait-initialized.sh"
       WAIT_SERVICES: "redis:6379 database:5432"
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
       REPORTING_DIR: '/mnt/reports/'
     configs:
       - source: init-db
@@ -198,3 +226,11 @@ volumes:
   report-data:
 networks:
   intnet:
+secrets:
+  OS_PASSWORD:
+    external: true
+  OS_PASSWORD_DOWNLOAD:
+    external: true
+  DJANGO_PASSWORD:
+    external: true
+  
\ No newline at end of file

docker-compose.emg.ops.yml

@@ -23,13 +23,13 @@ services:
         - "traefik.http.routers.emg-renderer-redirect.middlewares=redirect@file"
         - "traefik.http.routers.emg-renderer-redirect.entrypoints=http"
         # router for referrer based access (https)
-        - "traefik.http.routers.emg-renderer_referer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.emg-renderer_referer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.emg-renderer_referer.middlewares=compress@file,cors@file"
         - "traefik.http.routers.emg-renderer_referer.tls=true"
         - "traefik.http.routers.emg-renderer_referer.tls.certresolver=default"
         - "traefik.http.routers.emg-renderer_referer.entrypoints=https"
         # router for referrer based access (http)
-        - "traefik.http.routers.emg-renderer_referer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.emg-renderer_referer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.emg-renderer_referer-redirect.middlewares=redirect@file"
         - "traefik.http.routers.emg-renderer_referer-redirect.entrypoints=http"
         # general
@@ -65,13 +65,13 @@ services:
         - "traefik.http.routers.emg-cache-redirect.middlewares=redirect@file"
         - "traefik.http.routers.emg-cache-redirect.entrypoints=http"
         # router for referrer based access (https)
-        - "traefik.http.routers.emg-cache_referer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.emg-cache_referer.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.emg-cache_referer.middlewares=cache-stripprefix,compress@file,cors@file"
         - "traefik.http.routers.emg-cache_referer.tls=true"
         - "traefik.http.routers.emg-cache_referer.tls.certresolver=default"
         - "traefik.http.routers.emg-cache_referer.entrypoints=https"
         # router for referrer based access (http)
-        - "traefik.http.routers.emg-cache_referer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.emg-cache_referer-redirect.rule=Host(`emg.pdas.prism.eox.at`, `a.emg.pdas.prism.eox.at`, `b.emg.pdas.prism.eox.at`, `c.emg.pdas.prism.eox.at`, `d.emg.pdas.prism.eox.at`, `e.emg.pdas.prism.eox.at`, `f.emg.pdas.prism.eox.at`, `g.emg.pdas.prism.eox.at`, `h.emg.pdas.prism.eox.at`, `emg.pass.copernicus.eu`, `a.emg.pass.copernicus.eu`, `b.emg.pass.copernicus.eu`, `c.emg.pass.copernicus.eu`, `d.emg.pass.copernicus.eu`, `e.emg.pass.copernicus.eu`, `f.emg.pass.copernicus.eu`, `g.emg.pass.copernicus.eu`, `h.emg.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|emg.pdas.prism.eox.at|emg.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.emg-cache_referer-redirect.middlewares=redirect@file"
         - "traefik.http.routers.emg-cache_referer-redirect.entrypoints=http"
         # general

docker-compose.emg.yml

@@ -14,6 +14,11 @@ services:
         constraints: [node.role == manager]
     networks:
       - intnet
+    command: ["postgres", "-c", "max_connections=300"]
+    sysctls:
+      net.ipv4.tcp_keepalive_time: 600
+      net.ipv4.tcp_keepalive_probes: 5
+      net.ipv4.tcp_keepalive_intvl: 10
   redis:
     image: redis
     volumes:
@@ -36,6 +41,8 @@ services:
       - env/emg_db.env
       - env/emg_django.env
       - env/emg_obs.env
+    secrets:
+      - DJANGO_PASSWORD
     environment:
       INSTANCE_ID: "prism-view-server_renderer"
       INSTALL_DIR: "/var/www/pvs/dev/"
@@ -43,6 +50,7 @@ services:
       INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
       STARTUP_SCRIPTS: "/wait-initialized.sh"
       WAIT_SERVICES: "database:5432"
+      DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
     configs:
       - source: init-db
         target: /init-db.sh
@@ -61,11 +69,16 @@ services:
       - env/emg.env
       - env/emg_db.env
       - env/emg_obs.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_cache"
       RENDERER_HOST: renderer
       WAIT_SERVICES: "database:5432 renderer:80"
       WAIT_TIMEOUT: 300  # wait up to 5 minutes
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
     deploy:
       replicas: 1
     networks:
@@ -81,10 +94,15 @@ services:
       - env/emg.env
       - env/emg_obs.env
       - env/emg_redis.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_seeder"
       RENDERER_HOST: renderer
       WAIT_SERVICES: "redis:6379 database:5432"
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
     deploy:
       replicas: 0
     networks:
@@ -107,9 +125,14 @@ services:
       - env/emg.env
       - env/emg_obs.env
       - env/emg_redis.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_preprocessor"
       WAIT_SERVICES: "redis:6379"
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
     configs:
       - source: preprocessor-config
         target: /config.yaml      
@@ -135,6 +158,9 @@ services:
       - env/emg_db.env
       - env/emg_obs.env
       - env/emg_redis.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_registrar"
       INSTALL_DIR: "/var/www/pvs/dev/"
@@ -144,6 +170,9 @@ services:
       INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
       STARTUP_SCRIPTS: "/wait-initialized.sh"
       WAIT_SERVICES: "redis:6379 database:5432"
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
+      
       REPORTING_DIR: '/mnt/reports/'
     configs:
       - source: init-db
@@ -198,3 +227,10 @@ volumes:
   report-data:
 networks:
   intnet:
+secrets:
+  OS_PASSWORD:
+    external: true
+  OS_PASSWORD_DOWNLOAD:
+    external: true
+  DJANGO_PASSWORD:
+    external: true
\ No newline at end of file

docker-compose.vhr18.ops.yml

@@ -24,13 +24,13 @@ services:
         - "traefik.http.routers.vhr18-renderer-redirect.middlewares=redirect@file"
         - "traefik.http.routers.vhr18-renderer-redirect.entrypoints=http"
         # router for referrer based access (https)
-        - "traefik.http.routers.vhr18-renderer_referer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.vhr18-renderer_referer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.vhr18-renderer_referer.middlewares=compress@file,cors@file"
         - "traefik.http.routers.vhr18-renderer_referer.tls=true"
         - "traefik.http.routers.vhr18-renderer_referer.tls.certresolver=default"
         - "traefik.http.routers.vhr18-renderer_referer.entrypoints=https"
         # router for referrer based access (http)
-        - "traefik.http.routers.vhr18-renderer_referer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.vhr18-renderer_referer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/ows`, `/opensearch`, `/admin`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.vhr18-renderer_referer-redirect.middlewares=redirect@file"
         - "traefik.http.routers.vhr18-renderer_referer-redirect.entrypoints=http"
         # general
@@ -65,13 +65,13 @@ services:
         - "traefik.http.routers.vhr18-cache-redirect.middlewares=redirect@file"
         - "traefik.http.routers.vhr18-cache-redirect.entrypoints=http"
         # router for referrer based access (https)
-        - "traefik.http.routers.vhr18-cache_referer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.vhr18-cache_referer.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.vhr18-cache_referer.middlewares=cache-stripprefix,compress@file,cors@file"
         - "traefik.http.routers.vhr18-cache_referer.tls=true"
         - "traefik.http.routers.vhr18-cache_referer.tls.certresolver=default"
         - "traefik.http.routers.vhr18-cache_referer.entrypoints=https"
         # router for referrer based access (http)
-        - "traefik.http.routers.vhr18-cache_referer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
+        - "traefik.http.routers.vhr18-cache_referer-redirect.rule=Host(`vhr18.pdas.prism.eox.at`, `a.vhr18.pdas.prism.eox.at`, `b.vhr18.pdas.prism.eox.at`, `c.vhr18.pdas.prism.eox.at`, `d.vhr18.pdas.prism.eox.at`, `e.vhr18.pdas.prism.eox.at`, `f.vhr18.pdas.prism.eox.at`, `g.vhr18.pdas.prism.eox.at`, `h.vhr18.pdas.prism.eox.at`, `vhr18.pass.copernicus.eu`, `a.vhr18.pass.copernicus.eu`, `b.vhr18.pass.copernicus.eu`, `c.vhr18.pass.copernicus.eu`, `d.vhr18.pass.copernicus.eu`, `e.vhr18.pass.copernicus.eu`, `f.vhr18.pass.copernicus.eu`, `g.vhr18.pass.copernicus.eu`, `h.vhr18.pass.copernicus.eu`) && PathPrefix(`/cache`) && HeadersRegexp(`Referer`, `(https?://)?(panda.copernicus.eu|panda.cdsv3.eu|panda-demo.ondaprism.eu|panda-demo.copernicus.eu|cdsportal-demo.copernicus.eu|ocqc-demo.copernicus.eu|spdm-intservices.cds.esa.int|spdm-intservices-adm.cds.esa.int|vhr18.pdas.prism.eox.at|vhr18.pass.copernicus.eu)/?`)"
         - "traefik.http.routers.vhr18-cache_referer-redirect.middlewares=redirect@file"
         - "traefik.http.routers.vhr18-cache_referer-redirect.entrypoints=http"
         # general

docker-compose.vhr18.yml

@@ -14,6 +14,11 @@ services:
         constraints: [node.role == manager]
     networks:
       - intnet
+    command: ["postgres", "-c", "max_connections=300"]
+    sysctls:
+      net.ipv4.tcp_keepalive_time: 600
+      net.ipv4.tcp_keepalive_probes: 5
+      net.ipv4.tcp_keepalive_intvl: 10
   redis:
     image: redis
     volumes:
@@ -36,6 +41,8 @@ services:
       - env/vhr18_db.env
       - env/vhr18_django.env
       - env/vhr18_obs.env
+    secrets:
+      - DJANGO_PASSWORD
     environment:
       INSTANCE_ID: "prism-view-server_renderer"
       INSTALL_DIR: "/var/www/pvs/dev/"
@@ -43,6 +50,7 @@ services:
       INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
       STARTUP_SCRIPTS: "/wait-initialized.sh"
       WAIT_SERVICES: "database:5432"
+      DJANGO_PASSWORD_FILE: "/run/secrets/DJANGO_PASSWORD"
     configs:
       - source: init-db
         target: /init-db.sh
@@ -61,11 +69,16 @@ services:
       - env/vhr18.env
       - env/vhr18_db.env
       - env/vhr18_obs.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_cache"
       RENDERER_HOST: renderer
       WAIT_SERVICES: "database:5432 renderer:80"
       WAIT_TIMEOUT: 300  # wait up to 5 minutes
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
     deploy:
       replicas: 1
     logging:
@@ -84,10 +97,15 @@ services:
       - env/vhr18.env
       - env/vhr18_obs.env
       - env/vhr18_redis.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_seeder"
       RENDERER_HOST: renderer
       WAIT_SERVICES: "redis:6379 database:5432"
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
     deploy:
       replicas: 0
     networks:
@@ -110,12 +128,17 @@ services:
       - env/vhr18.env
       - env/vhr18_obs.env
       - env/vhr18_redis.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_preprocessor"
       WAIT_SERVICES: "redis:6379"
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
     configs:
       - source: preprocessor-config
-        target: /config.yaml
+        target: /config.yaml      
     deploy:
       replicas: 1
     networks:
@@ -138,6 +161,9 @@ services:
       - env/vhr18_db.env
       - env/vhr18_obs.env
       - env/vhr18_redis.env
+    secrets:
+      - OS_PASSWORD
+      - OS_PASSWORD_DOWNLOAD
     environment:
       INSTANCE_ID: "prism-view-server_registrar"
       INSTALL_DIR: "/var/www/pvs/dev/"
@@ -147,6 +173,8 @@ services:
       INIT_SCRIPTS: "/configure.sh /init-db.sh /initialized.sh"
       STARTUP_SCRIPTS: "/wait-initialized.sh"
       WAIT_SERVICES: "redis:6379 database:5432"
+      OS_PASSWORD_FILE: "/run/secrets/OS_PASSWORD"
+      OS_PASSWORD_DOWNLOAD_FILE: "/run/secrets/OS_PASSWORD_DOWNLOAD"
       REPORTING_DIR: '/mnt/reports/'
     configs:
       - source: init-db
@@ -177,7 +205,7 @@ services:
       replicas: 1
 
     ports:
-        - "2222:22"
+      - "2222:22"
   ingestor:
     image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
     deploy:
@@ -205,3 +233,10 @@ volumes:
   report-data:
 networks:
   intnet:
+secrets:
+  OS_PASSWORD:
+    external: true
+  OS_PASSWORD_DOWNLOAD:
+    external: true
+  DJANGO_PASSWORD:
+    external: true