diff --git a/core/Dockerfile b/core/Dockerfile
index da66d65d9dbc319b4d991d5bba9ae2279cb651cf..a49166790edc279a7aec1509ddb03a36f4071b50 100644
--- a/core/Dockerfile
+++ b/core/Dockerfile
@@ -50,6 +50,7 @@ ENV INSTANCE_ID="prism-view-server_core" \
INSTANCE_NAME="pvs_instance"\
INSTANCE_DIR="/var/www/pvs/dev/pvs_instance/" \
COLLECTION= \
+ UPLOAD_CONTAINER= \
DB_USER= \
DB_PW= \
DB_HOST= \
diff --git a/core/registrar.py b/core/registrar.py
index 51bb1892e6ab2845925b6e29ac0a68af0ebce259..a774a1d501fe24cbb537c0ff4e7701e5a952bc42 100644
--- a/core/registrar.py
+++ b/core/registrar.py
@@ -139,17 +139,15 @@ class RegistrationError(Exception):
@transaction.atomic
def registrar(
collection,
- objects_prefix, replace=False, client=None, registered_set_key=None
+ objects_prefix, upload_container, replace=False, client=None, registered_set_key=None
):
logger.info("Starting registration of product '%s'." % objects_prefix)
- container = objects_prefix.split("/")[1]
- package = "/".join(objects_prefix.split("/")[2:])
metadata_package, data_package, has_vrt = None, None, None
with SwiftService() as swift:
list_parts_gen = swift.list(
- container=container, options={"prefix": package},
+ container=upload_container, options={"prefix": objects_prefix},
)
for page in list_parts_gen:
if page["success"]:
@@ -184,7 +182,7 @@ def registrar(
product_type = data_package.split("/")[1]
product, replaced = ProductRegistrator().register(
- metadata_locations=[[container,
+ metadata_locations=[[upload_container,
metadata_package, ], ],
type_name="%s_Product_%s" % (collection, product_type),
replace=replace,
@@ -211,8 +209,8 @@ def registrar(
models.collection_insert_eo_object(collection_level_3, product)
report = GDALRegistrator().register(
- data_locations=[[container, data_package, ], ],
- metadata_locations=[[container,
+ data_locations=[[upload_container, data_package, ], ],
+ metadata_locations=[[upload_container,
metadata_package, ], ],
coverage_type_name="RGBNir",
overrides={"identifier": "%s__coverage" % product.identifier},
@@ -246,6 +244,7 @@ def registrar(
def registrar_redis_wrapper(
collection,
+ upload_container,
replace=False, host="localhost", port=6379,
register_queue_key="register_queue",
registered_set_key="registered_set",
@@ -260,6 +259,7 @@ def registrar_redis_wrapper(
registrar(
collection,
value[1],
+ upload_container,
replace=replace,
client=client,
registered_set_key=registered_set_key
@@ -332,15 +332,22 @@ if __name__ == "__main__":
logger.critical("Provided collection '%s' is not valid." % collection)
sys.exit(1)
+ upload_container = os.environ.get('UPLOAD_CONTAINER')
+ if upload_container is None:
+ logger.critical("UPLOAD_CONTAINER environment variable not set.")
+ sys.exit(1)
+
if arg_values.mode == "standard":
registrar(
collection,
arg_values.objects_prefix,
+ upload_container,
replace=arg_values.replace,
)
else:
registrar_redis_wrapper(
collection,
+ upload_container,
replace=arg_values.replace,
host=arg_values.redis_host,
port=arg_values.redis_port,
diff --git a/env/emg.env b/env/emg.env
index 75b1cd21157bbc350b10c8586afe561b1a939936..d056fa2885245bb91d5d250f4fed083dcc54eafe 100644
--- a/env/emg.env
+++ b/env/emg.env
@@ -1,4 +1,5 @@
COLLECTION=Emergency
+UPLOAD_CONTAINER=emg-data
GDAL_DISABLE_READDIR_ON_OPEN=TRUE
CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.TIF,.tif,.xml
diff --git a/env/vhr18.env b/env/vhr18.env
index 8c04b00bf71965ffbbdfcbebfe83df770577fdcb..33c93caf76fbc1d6cfc995e939bf40a66cfce66d 100644
--- a/env/vhr18.env
+++ b/env/vhr18.env
@@ -1,4 +1,5 @@
COLLECTION=VHR_IMAGE_2018
+UPLOAD_CONTAINER=vhr18-data
GDAL_DISABLE_READDIR_ON_OPEN=TRUE
CPL_VSIL_CURL_ALLOWED_EXTENSIONS=.TIF,.tif,.xml
diff --git a/preprocessor/Dockerfile b/preprocessor/Dockerfile
index 8a9a19969a7e92ee5f86744a061463f535bda9b0..39b317ed07e797dd0dcce2811e53ea32155fa7b7 100644
--- a/preprocessor/Dockerfile
+++ b/preprocessor/Dockerfile
@@ -41,6 +41,7 @@ RUN apk add --no-cache gcc py3-pip python3-dev py3-setuptools musl-dev \
ENV INSTANCE_ID="prism-data-access-server_preprocessor" \
COLLECTION= \
+ UPLOAD_CONTAINER= \
ST_AUTH_VERSION=3 \
OS_AUTH_URL="https://auth.cloud.ovh.net/v3/" \
OS_USERNAME= \
diff --git a/preprocessor/preprocessor.py b/preprocessor/preprocessor.py
index 58d08be7e152d837a50075e19aec8ecc007e0f1f..62d964dbb827404b728a81391227fb85fe45897a 100644
--- a/preprocessor/preprocessor.py
+++ b/preprocessor/preprocessor.py
@@ -86,7 +86,7 @@ def setup_logging(verbosity):
def preprocessor(
- collection, tar_object_path, replace=False,
+ collection, tar_object_path, upload_container, replace=False,
client=None, register_queue_key=None
):
logger.info("Starting preprocessing of '%s'." % (tar_object_path))
@@ -100,7 +100,7 @@ def preprocessor(
if not replace:
try:
list_parts_gen = swift.list(
- container=container, options={"prefix": package},
+ container=upload_container, options={"prefix": tar_object_path},
)
for page in list_parts_gen:
if page["success"]:
@@ -245,7 +245,7 @@ def preprocessor(
SwiftUploadObject(
vrt_name,
object_name=os.path.join(
- package, os.path.basename(vrt_name))
+ container, package, os.path.basename(vrt_name))
)
)
@@ -257,23 +257,26 @@ def preprocessor(
swift_upload_options["segment_size"] = 2 * 1024 * 1024 * 1024 # 2gb segments
dest_object_name = os.path.join(
- package, os.path.basename(data_file)
+ container, package, os.path.basename(data_file)
)
objects.append(
- SwiftUploadObject(data_file, object_name=dest_object_name)
+ SwiftUploadObject(
+ data_file,
+ object_name=dest_object_name
+ )
)
# add metadata to files to be uploaded after data files
objects.append(
SwiftUploadObject(
os.path.join(tmpdirname, metadata_file),
- object_name=os.path.join(package, metadata_file)
+ object_name=os.path.join(container, package, metadata_file)
)
)
# upload files
for upload in swift.upload(
- container=container,
+ container=upload_container,
objects=objects,
options=swift_upload_options
):
@@ -312,7 +315,7 @@ def preprocessor(
def preprocessor_redis_wrapper(
- collection, replace=False, host="localhost", port=6379,
+ collection, upload_container, replace=False, host="localhost", port=6379,
preprocess_queue_key="preprocess_queue",
register_queue_key="register_queue"
):
@@ -325,6 +328,7 @@ def preprocessor_redis_wrapper(
preprocessor(
collection,
value[1],
+ upload_container,
replace=replace,
client=client,
register_queue_key=register_queue_key
@@ -392,15 +396,22 @@ if __name__ == "__main__":
logger.critical("Provided collection '%s' is not valid." % collection)
sys.exit(1)
+ upload_container = os.environ.get('UPLOAD_CONTAINER')
+ if upload_container is None:
+ logger.critical("UPLOAD_CONTAINER environment variable not set.")
+ sys.exit(1)
+
if arg_values.mode == "standard":
preprocessor(
collection,
arg_values.tar_object_path,
+ upload_container,
replace=arg_values.replace,
)
else:
preprocessor_redis_wrapper(
collection,
+ upload_container,
replace=arg_values.replace,
host=arg_values.redis_host,
port=arg_values.redis_port,