Merge branch 'master' of gitlab.eox.at:esa/prism/vs

cache/run-seeder.sh

@@ -4,4 +4,4 @@
 
 echo "Running seeder"
 
-python3 /seeder.py ${COLLECTION} --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-queue-key ${REDIS_QUEUE_KEY}
+python3 /seeder.py --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-queue-key ${REDIS_QUEUE_KEY}

cache/seeder.py

@@ -44,12 +44,6 @@ import sqlite3
 import boto3
 
 
-# collection:
-COLLECTION_MAP = {
-    # TODO
-}
-
-
 logger = logging.getLogger(__name__)
 
 
@@ -224,12 +218,6 @@ if __name__ == "__main__":
     Pre-seeds cache of PRISM View Server (pvs).
     """)
 
-    parser.add_argument(
-        "collection", default=None,
-        help=(
-            "Collection the seeder is run for."
-        )
-    )
     parser.add_argument(
         "--mode", default="standard", choices=["standard", "redis"],
         help=(
@@ -278,9 +266,9 @@ if __name__ == "__main__":
 
     setup_logging(arg_values.verbosity)
 
-    collection = arg_values.collection
-    if collection not in COLLECTION_MAP:
-        logger.critical("Provided collection '%s' is not valid." % collection)
+    collection = os.environ.get('Collection')
+    if collection is None:
+        logger.critical("Collection environment variable not set.")
         sys.exit(1)
 
     if arg_values.mode == "standard":

core/Dockerfile

@@ -50,6 +50,7 @@ ENV INSTANCE_ID="prism-view-server_core" \
     INSTANCE_NAME="pvs_instance"\
     INSTANCE_DIR="/var/www/pvs/dev/pvs_instance/" \
     COLLECTION= \
+    UPLOAD_CONTAINER= \
     DB_USER= \
     DB_PW= \
     DB_HOST= \

core/registrar.py

@@ -60,13 +60,6 @@ from eoxserver.resources.coverages.registration.registrators.gdal import (
     GDALRegistrator
 )
 
-
-# collection: [name]
-COLLECTION_MAP = {
-    "VHR_IMAGE_2018": ["VHR IMAGE 2018", ],
-    "Emergency": ["Emergency", ],
-}
-
 logger = logging.getLogger(__name__)
 
 
@@ -139,17 +132,15 @@ class RegistrationError(Exception):
 @transaction.atomic
 def registrar(
     collection,
-    objects_prefix, replace=False, client=None, registered_set_key=None
+    objects_prefix, upload_container, replace=False, client=None, registered_set_key=None
 ):
     logger.info("Starting registration of product '%s'." % objects_prefix)
 
-    container = objects_prefix.split("/")[1]
-    package = "/".join(objects_prefix.split("/")[2:])
     metadata_package, data_package, has_vrt = None, None, None
 
     with SwiftService() as swift:
         list_parts_gen = swift.list(
-            container=container, options={"prefix": package},
+            container=upload_container, options={"prefix": objects_prefix},
         )
         for page in list_parts_gen:
             if page["success"]:
@@ -184,7 +175,7 @@ def registrar(
     product_type = data_package.split("/")[1]
 
     product, replaced = ProductRegistrator().register(
-        metadata_locations=[[container,
+        metadata_locations=[[upload_container,
                              metadata_package, ], ],
         type_name="%s_Product_%s" % (collection, product_type),
         replace=replace,
@@ -211,8 +202,8 @@ def registrar(
         models.collection_insert_eo_object(collection_level_3, product)
 
     report = GDALRegistrator().register(
-        data_locations=[[container, data_package, ], ],
-        metadata_locations=[[container,
+        data_locations=[[upload_container, data_package, ], ],
+        metadata_locations=[[upload_container,
                              metadata_package, ], ],
         coverage_type_name="RGBNir",
         overrides={"identifier": "%s__coverage" % product.identifier},
@@ -246,6 +237,7 @@ def registrar(
 
 def registrar_redis_wrapper(
     collection,
+    upload_container,
     replace=False, host="localhost", port=6379,
     register_queue_key="register_queue",
     registered_set_key="registered_set",
@@ -260,6 +252,7 @@ def registrar_redis_wrapper(
             registrar(
                 collection,
                 value[1],
+                upload_container,
                 replace=replace,
                 client=client,
                 registered_set_key=registered_set_key
@@ -275,13 +268,6 @@ if __name__ == "__main__":
     Register products.
     """)
 
-    parser.add_argument(
-        "collection", default=None,
-        help=(
-            "Collection the registrar is run for."
-        )
-    )
-
     parser.add_argument(
         "--mode", default="standard", choices=["standard", "redis"],
         help=(
@@ -326,21 +312,27 @@ if __name__ == "__main__":
 
     setup_logging(arg_values.verbosity)
 
-    collection = arg_values.collection
+    collection = os.environ.get('Collection')
+    if collection is None:
+        logger.critical("Collection environment variable not set.")
+        sys.exit(1)
 
-    if collection not in COLLECTION_MAP:
-        logger.critical("Provided collection '%s' is not valid." % collection)
+    upload_container = os.environ.get('UPLOAD_CONTAINER')
+    if upload_container is None:
+        logger.critical("UPLOAD_CONTAINER environment variable not set.")
         sys.exit(1)
 
     if arg_values.mode == "standard":
         registrar(
             collection,
             arg_values.objects_prefix,
+            upload_container,
             replace=arg_values.replace,
         )
     else:
         registrar_redis_wrapper(
             collection,
+            upload_container,
             replace=arg_values.replace,
             host=arg_values.redis_host,
             port=arg_values.redis_port,

core/run-registrar.sh

@@ -2,4 +2,4 @@
 
 echo "Running registrar"
 
- python3 /registrar.py ${COLLECTION} --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-register-queue-key ${REDIS_REGISTER_QUEUE_KEY} --redis-registered-set-key ${REDIS_REGISTERED_SET_KEY}
+ python3 /registrar.py --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-register-queue-key ${REDIS_REGISTER_QUEUE_KEY} --redis-registered-set-key ${REDIS_REGISTERED_SET_KEY}

env/emg.env

 COLLECTION=Emergency
+UPLOAD_CONTAINER=emg-data
 
 GDAL_DISABLE_READDIR_ON_OPEN=TRUE
 CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.TIF,.tif,.xml

env/vhr18.env

 COLLECTION=VHR_IMAGE_2018
+UPLOAD_CONTAINER=vhr18-data
 
 GDAL_DISABLE_READDIR_ON_OPEN=TRUE
 CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.TIF,.tif,.xml

preprocessor/Dockerfile

@@ -41,6 +41,7 @@ RUN apk add --no-cache gcc py3-pip python3-dev py3-setuptools musl-dev \
 
 ENV INSTANCE_ID="prism-data-access-server_preprocessor" \
     COLLECTION= \
+    UPLOAD_CONTAINER= \
     ST_AUTH_VERSION=3 \
     OS_AUTH_URL="https://auth.cloud.ovh.net/v3/" \
     OS_USERNAME= \

preprocessor/get_min_max.py

@@ -41,11 +41,6 @@ import re
 from swiftclient.service import SwiftError, SwiftService
 
 
-# collection: [name]
-COLLECTION_MAP = {
-    "VHR_IMAGE_2018": ["VHR IMAGE 2018", ],
-}
-
 logger = logging.getLogger(__name__)
 
 
@@ -146,13 +141,6 @@ if __name__ == "__main__":
     Get min/max of products.
     """)
 
-    parser.add_argument(
-        "collection", default=None,
-        help=(
-            "Collection the registrar is run for."
-        )
-    )
-
     parser.add_argument(
         "-v", "--verbosity", type=int, default=3, choices=[0, 1, 2, 3, 4],
         help=(
@@ -165,9 +153,9 @@ if __name__ == "__main__":
 
     setup_logging(arg_values.verbosity)
 
-    collection = arg_values.collection
-    if collection not in COLLECTION_MAP:
-        logger.critical("Provided collection '%s' is not valid." % collection)
+    collection = os.environ.get('Collection')
+    if collection is None:
+        logger.critical("Collection environment variable not set.")
         sys.exit(1)
 
     get_min_max(

preprocessor/preprocessor.py

@@ -46,11 +46,6 @@ from swiftclient.service import SwiftError, SwiftService, SwiftUploadObject
 
 import transform_chain
 
-# collection: [name]
-COLLECTION_MAP = {
-    "VHR_IMAGE_2018": ["VHR IMAGE 2018", ],
-    "Emergency": ["Emergency", ],
-}
 SPLIT_PARTS_CHECK = os.environ.get('SPLIT_PARTS_CHECK')
 ENFORCE_FOUR_BANDS = os.environ.get('ENFORCE_FOUR_BANDS')
 
@@ -86,7 +81,7 @@ def setup_logging(verbosity):
 
 
 def preprocessor(
-    collection, tar_object_path, replace=False,
+    collection, tar_object_path, upload_container, replace=False,
     client=None, register_queue_key=None
 ):
     logger.info("Starting preprocessing of '%s'." % (tar_object_path))
@@ -100,7 +95,7 @@ def preprocessor(
             if not replace:
                 try:
                     list_parts_gen = swift.list(
-                        container=container, options={"prefix": package},
+                        container=upload_container, options={"prefix": tar_object_path},
                     )
                     for page in list_parts_gen:
                         if page["success"]:
@@ -245,7 +240,7 @@ def preprocessor(
                     SwiftUploadObject(
                         vrt_name,
                         object_name=os.path.join(
-                            package, os.path.basename(vrt_name))
+                            container, package, os.path.basename(vrt_name))
                     )
                 )
 
@@ -257,23 +252,26 @@ def preprocessor(
                     swift_upload_options["segment_size"] = 2 * 1024 * 1024 * 1024  # 2gb segments
 
                 dest_object_name = os.path.join(
-                    package, os.path.basename(data_file)
+                    container, package, os.path.basename(data_file)
                 )
                 objects.append(
-                    SwiftUploadObject(data_file, object_name=dest_object_name)
+                    SwiftUploadObject(
+                        data_file,
+                        object_name=dest_object_name
+                    )
                 )
 
             # add metadata to files to be uploaded after data files
             objects.append(
                 SwiftUploadObject(
                     os.path.join(tmpdirname, metadata_file),
-                    object_name=os.path.join(package, metadata_file)
+                    object_name=os.path.join(container, package, metadata_file)
                 )
             )
 
             # upload files
             for upload in swift.upload(
-                container=container,
+                container=upload_container,
                 objects=objects,
                 options=swift_upload_options
             ):
@@ -312,7 +310,7 @@ def preprocessor(
 
 
 def preprocessor_redis_wrapper(
-    collection, replace=False, host="localhost", port=6379,
+    collection, upload_container, replace=False, host="localhost", port=6379,
     preprocess_queue_key="preprocess_queue",
     register_queue_key="register_queue"
 ):
@@ -325,6 +323,7 @@ def preprocessor_redis_wrapper(
         preprocessor(
             collection,
             value[1],
+            upload_container,
             replace=replace,
             client=client,
             register_queue_key=register_queue_key
@@ -337,12 +336,6 @@ if __name__ == "__main__":
     Preprocess product data.
     """)
 
-    parser.add_argument(
-        "collection", default=None,
-        help=(
-            "Collection the preprocessor is run for."
-        )
-    )
     parser.add_argument(
         "--mode", default="standard", choices=["standard", "redis"],
         help=(
@@ -387,20 +380,27 @@ if __name__ == "__main__":
 
     setup_logging(arg_values.verbosity)
 
-    collection = arg_values.collection
-    if collection not in COLLECTION_MAP:
-        logger.critical("Provided collection '%s' is not valid." % collection)
+    collection = os.environ.get('Collection')
+    if collection is None:
+        logger.critical("Collection environment variable not set.")
+        sys.exit(1)
+
+    upload_container = os.environ.get('UPLOAD_CONTAINER')
+    if upload_container is None:
+        logger.critical("UPLOAD_CONTAINER environment variable not set.")
         sys.exit(1)
 
     if arg_values.mode == "standard":
         preprocessor(
             collection,
             arg_values.tar_object_path,
+            upload_container,
             replace=arg_values.replace,
         )
     else:
         preprocessor_redis_wrapper(
             collection,
+            upload_container,
             replace=arg_values.replace,
             host=arg_values.redis_host,
             port=arg_values.redis_port,

preprocessor/run-preprocessor.sh

@@ -2,4 +2,4 @@
 
 echo "Running preprocessor"
 
-python3 /preprocessor.py ${COLLECTION} --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-preprocess-queue-key ${REDIS_PREPROCESS_QUEUE_KEY} --redis-register-queue-key ${REDIS_REGISTER_QUEUE_KEY}
+python3 /preprocessor.py --mode redis --redis-host ${REDIS_HOST} --redis-port ${REDIS_PORT} --redis-preprocess-queue-key ${REDIS_PREPROCESS_QUEUE_KEY} --redis-register-queue-key ${REDIS_REGISTER_QUEUE_KEY}