From 6f3ed31c1da68565b475b1fdcd2b94b14a683c74 Mon Sep 17 00:00:00 2001
From: Lubomir Bucek <lubomir.bucek@eox.at>
Date: Mon, 28 Sep 2020 21:13:34 +0200
Subject: [PATCH] change name of object when no bucket set
input dataxx/itemname gets uploaded to dataxx bucket as itemname (makes
it possible to mimic old preprocessor this way in case no bucket set)
motivation in this commit is to AVOID data duplication in new bucket and
dataxx buckets with possible reprocessing campaing
---
preprocessor/preprocessor/transfer/swift.py | 22 ++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/preprocessor/preprocessor/transfer/swift.py b/preprocessor/preprocessor/transfer/swift.py
index ad57dabc..a58f7fb1 100644
--- a/preprocessor/preprocessor/transfer/swift.py
+++ b/preprocessor/preprocessor/transfer/swift.py
@@ -39,19 +39,20 @@ class Base:
"os_user_domain_name": self.user_domain_name,
})
+ def validate_container(self, remote_dir):
+ if self.container:
+ # container was specified, use it
+ return self.container, remote_dir
+ # container needs to be extracted from path
+ # paths needs to be updated
+ return remote_dir.partition('/')[0], remote_dir.partition('/')[2]
+
class Downloader(Base):
""" Downloader for OpenStack swift object storages
"""
def download(self, remote_path: os.PathLike, local_path: os.PathLike) -> os.PathLike:
- if self.container:
- # container was specified, use it
- container = self.container
- else:
- # container needs to be extracted from path
- # paths needs to be updated
- container = remote_path.partition('/')[0]
- remote_path = remote_path.partition('/')[2]
+ container, remote_path = self.validate_container(remote_path)
target_filename = os.path.join(local_path, os.path.basename(remote_path))
with self.get_service() as swift:
results = swift.download(
@@ -74,6 +75,7 @@ class Uploader(Base):
"""
def upload(self, local_path: Union[os.PathLike, List[os.PathLike]], remote_dir: os.PathLike) -> List[os.PathLike]:
paths = local_path if isinstance(local_path, List) else [local_path]
+ container, remote_dir = self.validate_container(remote_dir)
remote_paths = [
os.path.join(
remote_dir,
@@ -99,7 +101,6 @@ class Uploader(Base):
with self.get_service() as swift:
# use container or first part of path
- container = self.container or remote_dir.partition('/')[0]
results = swift.upload(container=container, objects=objects, options=options)
for result in results:
@@ -123,8 +124,7 @@ class Uploader(Base):
def product_exists(self, remote_dir: os.PathLike) -> bool:
with self.get_service() as swift:
- # use container or first part of path
- container = self.container or remote_dir.partition('/')[0]
+ container, remote_dir = self.validate_container(remote_dir)
list_parts_gen = swift.list(
container=container, options={"prefix": remote_dir},
)
--
GitLab