Merge branch 'staging' into ingestor_test

README.md

@@ -101,7 +101,7 @@ The following services are defined via docker compose files.
             * holds ids for successfully registered items
             * `sadd` by registrar
         * register-failure_set
-            * holds its for failed registered items
+            * holds ids for failed registered items
             * `sadd` by registrar
     * seeding
         * seed_queue

cache/Dockerfile

@@ -47,9 +47,9 @@ ENV COLLECTION_ID= \
     APACHE_ServerName="pvs_cache" \
     APACHE_ServerAdmin="office@eox.at" \
     APACHE_NGEO_CACHE_ALIAS="/ows" \
-    REDIS_HOST= \
+    REDIS_HOST="redis" \
     REDIS_PORT="6379" \
-    REDIS_QUEUE_KEY="seed_queue" \
+    REDIS_SEED_QUEUE_KEY="seed_queue" \
     ST_AUTH_VERSION=3 \
     OS_AUTH_URL= \
     OS_USERNAME= \

cache/run-seeder.sh

@@ -4,4 +4,4 @@
 
 echo "Running seeder"
 
-python3 /seeder.py --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-queue-key ${REDIS_QUEUE_KEY}
+python3 /seeder.py --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-queue-key ${REDIS_SEED_QUEUE_KEY}

chart/values.yaml

@@ -49,12 +49,15 @@ config:
   redis:
     REDIS_PORT: "6379"
     REDIS_PREPROCESS_QUEUE_KEY: preprocess_queue
-    REDIS_QUEUE_KEY: seed_queue
+    REDIS_PREPROCESS_MD_QUEUE_KEY: preprocess-md_queue
+    REDIS_PREPROCESS_FAILURE_KEY: preprocess-failure_set
+    REDIS_PREPROCESS_PROGRESS_KEY: preprocessing_set
+    REDIS_PREPROCESS_SUCCESS_KEY: preprocess-success_set
     REDIS_REGISTER_QUEUE_KEY: register_queue
-    REDIS_REGISTERED_SET_KEY: registered_set
+    REDIS_REGISTER_FAILURE_KEY: register-failure_set
+    REDIS_REGISTER_PROGRESS_KEY: registering_set
+    REDIS_REGISTER_SUCCESS_KEY: register-success_set
     REDIS_SEED_QUEUE_KEY: seed_queue
-    REDIS_SET_KEY: registered_set
-
   client:
     layers: {}
       # VHR_IMAGE_2018_Level_1:

config/dem_preprocessor-config.yml

@@ -48,7 +48,7 @@ preprocessing:
           - BLOCKSIZE=512
           - COMPRESS=DEFLATE
           - NUM_THREADS=8
-          - BIGTIFF=IF_SAFER
+          - BIGTIFF=IF_NEEDED
           - OVERVIEWS=AUTO
   types:
     SAR_DGE_30: # just to pass validation

config/dem_registrar-config.yml

@@ -10,6 +10,7 @@ sources:
         auth_version: !env '${ST_AUTH_VERSION}'
         auth_url: !env '${OS_AUTH_URL}'
         auth_url_short: !env '${OS_AUTH_URL_SHORT}'
+        user_domain_name: !env '${OS_USER_DOMAIN_NAME}'
         container: !env '${UPLOAD_CONTAINER}'
 
 schemes:
@@ -21,8 +22,8 @@ backends:
   - type: eoxserver
     filter:
     kwargs:
-      instance_base_path: /var/www/pvs/dev
-      instance_name: pvs_instance
+      instance_base_path: !env '${INSTALL_DIR}'
+      instance_name: !env '${INSTANCE_NAME}'
       mapping:
         DEM1:
           COP-DEM_EEA-10-DGED:

config/emg_index-dev.html

@@ -39,15 +39,15 @@
         ],
         "constrainTimeDomain": true,
         "displayTimeDomain": [
-            "2019-01-01T00:00:00Z",
-            "2019-12-31T23:59:59Z",
+            "2019-06-01T00:00:00Z",
+            "2020-12-31T23:59:59Z",
         ],
         "displayInterval": "P1096D",
         "selectedTimeDomain": [
-            "2019-01-01T00:00:00Z",
-            "2019-12-31T23:59:59Z",
+            "2019-06-01T00:00:00Z",
+            "2020-12-31T23:59:59Z",
         ],
-        "selectableInterval": "P1096D",
+        "selectableInterval": "P10960D",
         "timeSliderControls": true,
         "maxTooltips": 1,
         "center": [12, 49],

config/emg_index-ops.html

@@ -59,15 +59,15 @@
         ],
         "constrainTimeDomain": true,
         "displayTimeDomain": [
-            "2019-01-01T00:00:00Z",
-            "2019-12-31T23:59:59Z",
+            "2019-06-01T00:00:00Z",
+            "2020-12-31T23:59:59Z",
         ],
         "displayInterval": "P1096D",
         "selectedTimeDomain": [
-            "2019-01-01T00:00:00Z",
-            "2019-12-31T23:59:59Z",
+            "2019-06-01T00:00:00Z",
+            "2020-12-31T23:59:59Z",
         ],
-        "selectableInterval": "P1096D",
+        "selectableInterval": "P10960D",
         "timeSliderControls": true,
         "maxTooltips": 1,
         "center": [12, 49],

config/emg_init-db.sh

@@ -921,7 +921,109 @@ if python3 manage.py id check "${COLLECTION}"; then
             --blue-nodata 0
         python3 manage.py browsetype create "${COLLECTION}"_Product_PH1B "NDVI" --traceback \
             --grey "(nir-red)/(nir+red)" --grey-range -1 1
-
+        python3 manage.py producttype create "${COLLECTION}"_Product_SK00 --traceback \
+            --coverage-type "RGBNir"
+        python3 manage.py browsetype create "${COLLECTION}"_Product_SK00 --traceback \
+            --red "red" \
+            --green "green" \
+            --blue "blue" \
+            --red-range 1 600 \
+            --green-range 1 600 \
+            --blue-range 1 600 \
+            --red-nodata 0 \
+            --green-nodata 0 \
+            --blue-nodata 0
+        python3 manage.py browsetype create "${COLLECTION}"_Product_SK00 "TRUE_COLOR" --traceback \
+            --red "red" \
+            --green "green" \
+            --blue "blue" \
+            --red-range 1 600 \
+            --green-range 1 600 \
+            --blue-range 1 600 \
+            --red-nodata 0 \
+            --green-nodata 0 \
+            --blue-nodata 0
+        python3 manage.py browsetype create "${COLLECTION}"_Product_SK00 "FALSE_COLOR" --traceback \
+            --red "nir" \
+            --green "red" \
+            --blue "green" \
+            --red-range 1 600 \
+            --green-range 1 600 \
+            --blue-range 1 600 \
+            --red-nodata 0 \
+            --green-nodata 0 \
+            --blue-nodata 0
+        python3 manage.py browsetype create "${COLLECTION}"_Product_SK00 "NDVI" --traceback \
+            --grey "(nir-red)/(nir+red)" --grey-range -1 1
+        python3 manage.py producttype create "${COLLECTION}"_Product_SW03 --traceback \
+            --coverage-type "RGBNir"
+        python3 manage.py browsetype create "${COLLECTION}"_Product_SW03 --traceback \
+            --red "red" \
+            --green "green" \
+            --blue "blue" \
+            --red-range 1 1000 \
+            --green-range 1 1000 \
+            --blue-range 1 1000 \
+            --red-nodata 0 \
+            --green-nodata 0 \
+            --blue-nodata 0
+        python3 manage.py browsetype create "${COLLECTION}"_Product_SW03 "TRUE_COLOR" --traceback \
+            --red "red" \
+            --green "green" \
+            --blue "blue" \
+            --red-range 1 1000 \
+            --green-range 1 1000 \
+            --blue-range 1 1000 \
+            --red-nodata 0 \
+            --green-nodata 0 \
+            --blue-nodata 0
+        python3 manage.py browsetype create "${COLLECTION}"_Product_SW03 "FALSE_COLOR" --traceback \
+            --red "nir" \
+            --green "red" \
+            --blue "green" \
+            --red-range 1 1000 \
+            --green-range 1 1000 \
+            --blue-range 1 1000 \
+            --red-nodata 0 \
+            --green-nodata 0 \
+            --blue-nodata 0
+        python3 manage.py browsetype create "${COLLECTION}"_Product_SW03 "NDVI" --traceback \
+            --grey "(nir-red)/(nir+red)" --grey-range -1 1
+        # bands go in order b,g,r,nir and I did not change them in preprocessing
+        python3 manage.py producttype create "${COLLECTION}"_Product_PL00 --traceback \
+            --coverage-type "RGBNir"
+        python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 --traceback \
+            --red "blue" \
+            --green "green" \
+            --blue "red" \
+            --red-range 1500 15000 \
+            --green-range 1500 15000 \
+            --blue-range 1500 15000 \
+            --red-nodata 0 \
+            --green-nodata 0 \
+            --blue-nodata 0
+        python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 "TRUE_COLOR" --traceback \
+            --red "blue" \
+            --green "green" \
+            --blue "red" \
+            --red-range 1500 15000 \
+            --green-range 1500 15000 \
+            --blue-range 1500 15000 \
+            --red-nodata 0 \
+            --green-nodata 0 \
+            --blue-nodata 0
+        python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 "FALSE_COLOR" --traceback \
+            --red "nir" \
+            --green "blue" \
+            --blue "green" \
+            --red-range 1500 20000 \
+            --green-range 1500 15000 \
+            --blue-range 1500 15000 \
+            --red-nodata 0 \
+            --green-nodata 0 \
+            --blue-nodata 0
+        python3 manage.py browsetype create "${COLLECTION}"_Product_PL00 "NDVI" --traceback \
+            --grey "(nir-blue)/(nir+blue)" --grey-range -1 1
         python3 manage.py collectiontype create "${COLLECTION}"_Collection --traceback \
             --coverage-type "RGBNir" \
             --coverage-type "RGNirByte" \
@@ -967,6 +1069,9 @@ if python3 manage.py id check "${COLLECTION}"; then
             --product-type "${COLLECTION}"_Product_SP05 \
             --product-type "${COLLECTION}"_Product_SP06 \
             --product-type "${COLLECTION}"_Product_SP07 \
+            --product-type "${COLLECTION}"_Product_SK00 \
+            --product-type "${COLLECTION}"_Product_SW03 \
+            --product-type "${COLLECTION}"_Product_PL00 \
             --product-type "${COLLECTION}"_Product_KS03
 
         python3 manage.py collection create "${COLLECTION}" --type "${COLLECTION}"_Collection --traceback

config/emg_preprocessor-config.yml

@@ -35,23 +35,202 @@ level_extractor:
 preprocessing:
   defaults:
     move_files: true
+    nested: true
     data_file_globs:
       - '*.tif'
       - '*.jp2'
+      - '*.h5'
     output:
       options:
         format: COG
         dstSRS: 'EPSG:4326'
         dstNodata: 0
+        multithread: True
+        warpMemoryLimit: 3000
         creationOptions:
           - BLOCKSIZE=512
           - COMPRESS=DEFLATE
           - NUM_THREADS=8
-          - BIGTIFF=IF_SAFER
+          - BIGTIFF=YES
           - OVERVIEWS=AUTO
   types:
-    PH1B: # just to pass validation
-      nested: true 
+    KS03:
+      data_file_globs:
+        - "*.tif"
+      additional_file_globs:
+        - "*.rpc*"
+      stack_bands:
+        group_by: ".*/(.*)_P..tif"
+        sort_by: ".*_P(R|G|B|N).tif"
+        order:
+          - R
+          - G
+          - B
+          - N
+    RS02:
+      data_file_globs:
+        - "*imagery_*.tif"
+      georeference:
+        geotransforms:
+          - type: gcp
+          - type: no_op
+      calc:
+        formulas:
+          # complex b1/b2 to b1 uint16 for all 4 polarizations
+          - inputs:
+              A:
+                glob: '*HH.tif'
+                band: 1
+              B:
+                glob: '*HH.tif'
+                band: 2
+            formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
+            output_postfix: _proc
+            nodata_value: 0
+            data_type: UInt16
+          - inputs:
+              A:
+                glob: '*HV.tif'
+                band: 1
+              B:
+                glob: '*HV.tif'
+                band: 2
+            formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
+            output_postfix: _proc
+            nodata_value: 0
+            data_type: UInt16
+          - inputs:
+              A:
+                glob: '*VH.tif'
+                band: 1
+              B:
+                glob: '*VH.tif'
+                band: 2
+            formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
+            output_postfix: _proc
+            nodata_value: 0
+            data_type: UInt16
+          - inputs:
+              A:
+                glob: '*VV.tif'
+                band: 1
+              B:
+                glob: '*VV.tif'
+                band: 2
+            formula: sqrt(A.astype(float)*A.astype(float)+B.astype(float)*B.astype(float))
+            output_postfix: _proc
+            nodata_value: 0
+            data_type: UInt16
+      stack_bands:
+        sort_by: ".*_(HH|HV|VH|VV)_proc.tif"
+        order:
+          - HH
+          - HV
+          - VH
+          - VV
+    SK00:
+      data_file_globs:
+        - "*pansharpened_clip.tif"
+        - "*_pansharpened.tif"
+    SP06:
+      data_file_globs:
+        # throw away Panchromatic *_P_*
+        - "*IMG_*_PMS_*.JP2"
+        - "*IMG_*_PMS_*.tif"
+        - "*IMG_*_MS_*.JP2"
+        - "*IMG_*_MS_*.tif"
+      additional_file_globs:
+        - "*RPC_*"
+        - "*DIM_*"
+      georeference:
+        geotransforms:
+          - type: rpc
+          - type: no_op
+    SP07:
+      data_file_globs:
+        - "*IMG_*_PMS_*.JP2"
+        - "*IMG_*_PMS_*.tif"
+        - "*IMG_*_MS_*.JP2"
+        - "*IMG_*_MS_*.tif"
+      additional_file_globs:
+        - "*RPC_*"
+        - "*DIM_*"
+      georeference:
+        geotransforms:
+          # first try RPC, if not present, go on
+          - type: rpc
+          - type: no_op
+    PH1A:
+      data_file_globs:
+        - "*IMG_*_PMS_*.JP2"
+        - "*IMG_*_PMS_*.tif"
+        - "*IMG_*_MS_*.JP2"
+        - "*IMG_*_MS_*.tif"
+      additional_file_globs:
+        - "*RPC_*"
+        - "*DIM_*"
+      georeference:
+        geotransforms:
+          - type: rpc
+          - type: no_op
+    PH1B:
+      data_file_globs:
+        - "*IMG_*_PMS_*.JP2"
+        - "*IMG_*_PMS_*.tif"
+        - "*IMG_*_MS_*.JP2"
+        - "*IMG_*_MS_*.tif"
+      additional_file_globs:
+        - "*RPC_*"
+        - "*DIM_*"
+      georeference:
+        geotransforms:
+          - type: rpc
+          - type: no_op
+    PL00:
+      data_file_globs:
+        - "*MS.tif"
+        - "*MS_clip.tif"
+    SW03:
+      data_file_globs:
+        - "*.tif"
+    GY01:
+      # throw away Panchromatic *-P3D*
+      data_file_globs:
+        - "*-M3D*.TIF"
+        - "*-M3D*.tif"
+        - "*-S3D*.TIF"
+        - "*-S3D*.tif"
+        - "*-M2A*.TIF"
+        - "*-M2A*.tif"
+        - "*-S2A*.TIF"
+        - "*-S2A*.tif"
+    EW03:
+      data_file_globs:
+        - "*-M3D*.TIF"
+        - "*-M3D*.tif"
+        - "*-S3D*.TIF"
+        - "*-S3D*.tif"
+        - "*-M2A*.TIF"
+        - "*-M2A*.tif"
+        - "*-S2A*.TIF"
+        - "*-S2A*.tif"
+    EW02:
+      data_file_globs:
+        - "*-M3D*.TIF"
+        - "*-M3D*.tif"
+        - "*-S3D*.TIF"
+        - "*-S3D*.tif"
+        - "*-M2A*.TIF"
+        - "*-M2A*.tif"
+        - "*-S2A*.TIF"
+        - "*-S2A*.tif"
+    EW01:
+      data_file_globs:
+        - "*.TIF"
+        - "*.tif"
+    DM02:
+      data_file_globs:
+        - "*.tif"
 # this configuration is still a stub - not all product types are done 
 # https://gitlab.eox.at/esa/prism/vs/-/issues/56
 # https://gitlab.eox.at/esa/prism/vs/-/issues/23

config/emg_registrar-config.yml

@@ -10,6 +10,7 @@ sources:
         auth_version: !env '${ST_AUTH_VERSION}'
         auth_url: !env '${OS_AUTH_URL}'
         auth_url_short: !env '${OS_AUTH_URL_SHORT}'
+        user_domain_name: !env '${OS_USER_DOMAIN_NAME}'
         container: !env '${UPLOAD_CONTAINER}'
 
 schemes:
@@ -19,8 +20,8 @@ backends:
   - type: eoxserver
     filter:
     kwargs:
-      instance_base_path: /var/www/pvs/dev
-      instance_name: pvs_instance
+      instance_base_path: !env '${INSTALL_DIR}'
+      instance_name: !env '${INSTANCE_NAME}'
       mapping:
         CS00:
           ~:
@@ -246,6 +247,27 @@ backends:
               - !env '${COLLECTION}'
             coverages:
               SP07: RGBNir
+        SK00:
+          ~:
+            product_type_name: !env '${COLLECTION}_Product_SK00'
+            collections:
+              - !env '${COLLECTION}'
+            coverages:
+              SK00: RGBNir
+        SW03:
+          ~:
+            product_type_name: !env '${COLLECTION}_Product_SW03'
+            collections:
+              - !env '${COLLECTION}'
+            coverages:
+              SW03: RGBNir
+        PL00:
+          ~:
+            product_type_name: !env '${COLLECTION}_Product_PL00'
+            collections:
+              - !env '${COLLECTION}'
+            coverages:
+              PL00: RGBNir
         TX01_2:
           ~:
             product_type_name: !env '${COLLECTION}_Product_TX01_2'

config/vhr18_preprocessor-config.yml

@@ -46,7 +46,7 @@ preprocessing:
           - BLOCKSIZE=512
           - COMPRESS=DEFLATE
           - NUM_THREADS=8
-          - BIGTIFF=IF_SAFER
+          - BIGTIFF=IF_NEEDED
           - OVERVIEWS=AUTO
   types:
     PH1B: # just to pass validation

config/vhr18_registrar-config.yml

@@ -10,6 +10,7 @@ sources:
         auth_version: !env '${ST_AUTH_VERSION}'
         auth_url: !env '${OS_AUTH_URL}'
         auth_url_short: !env '${OS_AUTH_URL_SHORT}'
+        user_domain_name: !env '${OS_USER_DOMAIN_NAME}'
         container: !env '${UPLOAD_CONTAINER}'
 
 schemes:
@@ -19,8 +20,8 @@ backends:
   - type: eoxserver
     filter:
     kwargs:
-      instance_base_path: /var/www/pvs/dev
-      instance_name: pvs_instance
+      instance_base_path: !env '${INSTALL_DIR}'
+      instance_name: !env '${INSTANCE_NAME}'
       mapping:
         PL00:
           Level_1:

core/Dockerfile

@@ -65,10 +65,12 @@ ENV INSTANCE_ID="prism-view-server_core" \
     APACHE_ServerName="pvs_instance" \
     APACHE_ServerAdmin="office@eox.at" \
     APACHE_ALIAS="pvs" \
-    REDIS_HOST= \
-    REDIS_PORT= \
-    REDIS_REGISTER_QUEUE_KEY= \
-    REDIS_REGISTERED_SET_KEY= \
+    REDIS_HOST="redis" \
+    REDIS_PORT="6379" \
+    REDIS_REGISTER_QUEUE_KEY="register_queue" \
+    REDIS_REGISTER_FAILURE_KEY="register-failure_set" \
+    REDIS_REGISTER_PROGRESS_KEY="registering_set" \
+    REDIS_REGISTER_SUCCESS_KEY="register-success_set" \
     INIT_SCRIPTS="/configure.sh" \
     COLLECT_STATIC="false" \
     REGISTRAR_REPLACE=

core/configure.sh

@@ -28,6 +28,7 @@
 
     sed -e "s,http_service_url=http://localhost:8000/ows,http_service_url=${APACHE_ALIAS}/ows," -i pvs_instance/conf/eoxserver.conf
     sed -e "s/resampling_method=average/resampling_method=near/" -i pvs_instance/conf/eoxserver.conf
+    sed -e "s/image\/jp2,GDAL\/JPEG2000/#image\/jp2,GDAL\/JPEG2000/" -i pvs_instance/conf/formats.conf
     # TODO maxsize...
 
     echo "EOXS_VALIDATE_IDS_NCNAME = False" >> pvs_instance/settings.py

core/registrar/cli.py

@@ -53,14 +53,16 @@ def cli():
 @click.option('--host', type=str)
 @click.option('--port', type=int)
 @click.option('--listen-queue', type=str)
-@click.option('--registered-set-key', type=str)
+@click.option('--progress-set', type=str)
+@click.option('--failure-set', type=str)
+@click.option('--success-set', type=str)
 @click.option('--debug/--no-debug', default=False)
-def daemon(config_file=None, validate=False, replace=False, host=None, port=None, listen_queue=None, registered_set_key=None, debug=False):
+def daemon(config_file=None, validate=False, replace=False, host=None, port=None, listen_queue=None, progress_set=None, failure_set=None, success_set=None, debug=False):
     setup_logging(debug)
     config = load_config(config_file)
     if validate:
         validate_config(config)
-    run_daemon(config, replace, host, port, listen_queue, registered_set_key)
+    run_daemon(config, replace, host, port, listen_queue, progress_set, failure_set, success_set)
 
 
 @cli.command(help='Run a single, one-off registration')

core/registrar/daemon.py

@@ -9,10 +9,9 @@ from .registrar import register_file
 logger = logging.getLogger(__name__)
 
 
-def run_daemon(config, replace, host, port, listen_queue, registered_set_key):
+def run_daemon(config, replace, host, port, listen_queue, progress_set, failure_set, success_set):
     """ Run the registrar daemon, listening on a redis queue
-        for files to be registered. After preprocessing the filename
-        of the registered files will be pushed to the output queue.
+        for files to be registered.
     """
     # initialize the queue client
     client = redis.Redis(
@@ -22,10 +21,15 @@ def run_daemon(config, replace, host, port, listen_queue, registered_set_key):
     while True:
         # fetch an item from the queue to be registered
         _, value = client.brpop(listen_queue)
+        client.sadd(progress_set, value)
         # start the registration on that file
         try:
             item = register_file(config, value, replace)
-            client.sadd(registered_set_key, item.identifier)
-
+            client.sadd(success_set, item.identifier)
+            client.srem(progress_set, value)
         except Exception as e:
+            if 'is already registered' not in "%s" % e:
+                # do not add to failure if skipped due to already registered
+                client.sadd(failure_set, value)
+            client.srem(progress_set, value)
             logger.exception(e)

core/run-registrar.sh

@@ -11,5 +11,7 @@ registrar daemon \
     --host ${REDIS_HOST} \
     --port ${REDIS_PORT} \
     --listen-queue ${REDIS_REGISTER_QUEUE_KEY} \
-    --registered-set-key ${REDIS_REGISTERED_SET_KEY} \
+    --progress-set ${REDIS_REGISTER_PROGRESS_KEY} \
+    --failure-set ${REDIS_REGISTER_FAILURE_KEY} \
+    --success-set ${REDIS_REGISTER_SUCCESS_KEY} \
     ${replace} >&2

docker-compose.dem.yml

@@ -92,7 +92,6 @@ services:
     env_file:
       - env/dem.env
       - env/dem_obs.env
-      - env/dem_redis.env
     secrets:
       - OS_PASSWORD
     environment:
@@ -111,7 +110,6 @@ services:
     env_file:
       - env/dem.env
       - env/dem_obs.env
-      - env/dem_redis.env
     secrets:
       - OS_PASSWORD
       - OS_PASSWORD_DOWNLOAD
@@ -144,7 +142,6 @@ services:
       - env/dem.env
       - env/dem_db.env
       - env/dem_obs.env
-      - env/dem_redis.env
     secrets:
       - DJANGO_PASSWORD
       - OS_PASSWORD
@@ -190,8 +187,6 @@ services:
       replicas: 1
   ingestor:
     image: registry.gitlab.eox.at/esa/prism/vs/pvs_ingestor:latest
-    env_file:
-      - env/dem_redis.env
     command:
       ["python3", "/filedaemon.py"]
     volumes:

docker-compose.emg.yml

@@ -2,7 +2,8 @@ version: "3.6"
 services:
   database:
     image: mdillon/postgis:10
-    # PUT BACK VOLUME AFTER TEST
+    volumes:
+      - db-data:/var/lib/postgresql/data
     env_file:
       - env/emg.env
       - env/emg_db.env
@@ -91,7 +92,6 @@ services:
     env_file:
       - env/emg.env
       - env/emg_obs.env
-      - env/emg_redis.env
     secrets:
       - OS_PASSWORD
     environment:
@@ -113,8 +113,6 @@ services:
       - type: volume
         source: from-fepd
         target: /mnt/data
-    env_file:
-      - env/emg_redis.env
     environment:
       INSTANCE_ID: "prism-view-server_ingestor"
     deploy:
@@ -126,7 +124,6 @@ services:
     env_file:
       - env/emg.env
       - env/emg_obs.env
-      - env/emg_redis.env
     secrets:
       - OS_PASSWORD
       - OS_PASSWORD_DOWNLOAD
@@ -159,7 +156,6 @@ services:
       - env/emg.env
       - env/emg_db.env
       - env/emg_obs.env
-      - env/emg_redis.env
     secrets:
       - DJANGO_PASSWORD
       - OS_PASSWORD