use bucket name from env for preprocessor and registrar

core/Dockerfile

@@ -50,6 +50,7 @@ ENV INSTANCE_ID="prism-view-server_core" \
     INSTANCE_NAME="pvs_instance"\
     INSTANCE_DIR="/var/www/pvs/dev/pvs_instance/" \
     COLLECTION= \
+    UPLOAD_CONTAINER= \
     DB_USER= \
     DB_PW= \
     DB_HOST= \

core/registrar.py

@@ -139,17 +139,15 @@ class RegistrationError(Exception):
 @transaction.atomic
 def registrar(
     collection,
-    objects_prefix, replace=False, client=None, registered_set_key=None
+    objects_prefix, upload_container, replace=False, client=None, registered_set_key=None
 ):
     logger.info("Starting registration of product '%s'." % objects_prefix)
 
-    container = objects_prefix.split("/")[1]
-    package = "/".join(objects_prefix.split("/")[2:])
     metadata_package, data_package, has_vrt = None, None, None
 
     with SwiftService() as swift:
         list_parts_gen = swift.list(
-            container=container, options={"prefix": package},
+            container=upload_container, options={"prefix": objects_prefix},
         )
         for page in list_parts_gen:
             if page["success"]:
@@ -184,7 +182,7 @@ def registrar(
     product_type = data_package.split("/")[1]
 
     product, replaced = ProductRegistrator().register(
-        metadata_locations=[[container,
+        metadata_locations=[[upload_container,
                              metadata_package, ], ],
         type_name="%s_Product_%s" % (collection, product_type),
         replace=replace,
@@ -211,8 +209,8 @@ def registrar(
         models.collection_insert_eo_object(collection_level_3, product)
 
     report = GDALRegistrator().register(
-        data_locations=[[container, data_package, ], ],
-        metadata_locations=[[container,
+        data_locations=[[upload_container, data_package, ], ],
+        metadata_locations=[[upload_container,
                              metadata_package, ], ],
         coverage_type_name="RGBNir",
         overrides={"identifier": "%s__coverage" % product.identifier},
@@ -246,6 +244,7 @@ def registrar(
 
 def registrar_redis_wrapper(
     collection,
+    upload_container,
     replace=False, host="localhost", port=6379,
     register_queue_key="register_queue",
     registered_set_key="registered_set",
@@ -260,6 +259,7 @@ def registrar_redis_wrapper(
             registrar(
                 collection,
                 value[1],
+                upload_container,
                 replace=replace,
                 client=client,
                 registered_set_key=registered_set_key
@@ -332,15 +332,22 @@ if __name__ == "__main__":
         logger.critical("Provided collection '%s' is not valid." % collection)
         sys.exit(1)
 
+    upload_container = os.environ.get('UPLOAD_CONTAINER')
+    if upload_container is None:
+        logger.critical("UPLOAD_CONTAINER environment variable not set.")
+        sys.exit(1)
+
     if arg_values.mode == "standard":
         registrar(
             collection,
             arg_values.objects_prefix,
+            upload_container,
             replace=arg_values.replace,
         )
     else:
         registrar_redis_wrapper(
             collection,
+            upload_container,
             replace=arg_values.replace,
             host=arg_values.redis_host,
             port=arg_values.redis_port,

env/emg.env

 COLLECTION=Emergency
+UPLOAD_CONTAINER=emg-data
 
 GDAL_DISABLE_READDIR_ON_OPEN=TRUE
 CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.TIF,.tif,.xml

env/vhr18.env

 COLLECTION=VHR_IMAGE_2018
+UPLOAD_CONTAINER=vhr18-data
 
 GDAL_DISABLE_READDIR_ON_OPEN=TRUE
 CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.TIF,.tif,.xml

preprocessor/Dockerfile

@@ -41,6 +41,7 @@ RUN apk add --no-cache gcc py3-pip python3-dev py3-setuptools musl-dev \
 
 ENV INSTANCE_ID="prism-data-access-server_preprocessor" \
     COLLECTION= \
+    UPLOAD_CONTAINER= \
     ST_AUTH_VERSION=3 \
     OS_AUTH_URL="https://auth.cloud.ovh.net/v3/" \
     OS_USERNAME= \

preprocessor/preprocessor.py

@@ -86,7 +86,7 @@ def setup_logging(verbosity):
 
 
 def preprocessor(
-    collection, tar_object_path, replace=False,
+    collection, tar_object_path, upload_container, replace=False,
     client=None, register_queue_key=None
 ):
     logger.info("Starting preprocessing of '%s'." % (tar_object_path))
@@ -100,7 +100,7 @@ def preprocessor(
             if not replace:
                 try:
                     list_parts_gen = swift.list(
-                        container=container, options={"prefix": package},
+                        container=upload_container, options={"prefix": tar_object_path},
                     )
                     for page in list_parts_gen:
                         if page["success"]:
@@ -245,7 +245,7 @@ def preprocessor(
                     SwiftUploadObject(
                         vrt_name,
                         object_name=os.path.join(
-                            package, os.path.basename(vrt_name))
+                            container, package, os.path.basename(vrt_name))
                     )
                 )
 
@@ -257,23 +257,26 @@ def preprocessor(
                     swift_upload_options["segment_size"] = 2 * 1024 * 1024 * 1024  # 2gb segments
 
                 dest_object_name = os.path.join(
-                    package, os.path.basename(data_file)
+                    container, package, os.path.basename(data_file)
                 )
                 objects.append(
-                    SwiftUploadObject(data_file, object_name=dest_object_name)
+                    SwiftUploadObject(
+                        data_file,
+                        object_name=dest_object_name
+                    )
                 )
 
             # add metadata to files to be uploaded after data files
             objects.append(
                 SwiftUploadObject(
                     os.path.join(tmpdirname, metadata_file),
-                    object_name=os.path.join(package, metadata_file)
+                    object_name=os.path.join(container, package, metadata_file)
                 )
             )
 
             # upload files
             for upload in swift.upload(
-                container=container,
+                container=upload_container,
                 objects=objects,
                 options=swift_upload_options
             ):
@@ -312,7 +315,7 @@ def preprocessor(
 
 
 def preprocessor_redis_wrapper(
-    collection, replace=False, host="localhost", port=6379,
+    collection, upload_container, replace=False, host="localhost", port=6379,
     preprocess_queue_key="preprocess_queue",
     register_queue_key="register_queue"
 ):
@@ -325,6 +328,7 @@ def preprocessor_redis_wrapper(
         preprocessor(
             collection,
             value[1],
+            upload_container,
             replace=replace,
             client=client,
             register_queue_key=register_queue_key
@@ -392,15 +396,22 @@ if __name__ == "__main__":
         logger.critical("Provided collection '%s' is not valid." % collection)
         sys.exit(1)
 
+    upload_container = os.environ.get('UPLOAD_CONTAINER')
+    if upload_container is None:
+        logger.critical("UPLOAD_CONTAINER environment variable not set.")
+        sys.exit(1)
+
     if arg_values.mode == "standard":
         preprocessor(
             collection,
             arg_values.tar_object_path,
+            upload_container,
             replace=arg_values.replace,
         )
     else:
         preprocessor_redis_wrapper(
             collection,
+            upload_container,
             replace=arg_values.replace,
             host=arg_values.redis_host,
             port=arg_values.redis_port,