EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 6f3ed31c authored by Lubomir Dolezal's avatar Lubomir Dolezal
Browse files

change name of object when no bucket set

input dataxx/itemname gets uploaded to dataxx bucket as itemname (makes 
it possible to mimic old preprocessor this way in case no bucket set)
motivation in this commit is to AVOID data duplication in new bucket and 
dataxx buckets with possible reprocessing campaing
parent dd01c9da
No related branches found
No related tags found
No related merge requests found
......@@ -39,19 +39,20 @@ class Base:
"os_user_domain_name": self.user_domain_name,
})
def validate_container(self, remote_dir):
if self.container:
# container was specified, use it
return self.container, remote_dir
# container needs to be extracted from path
# paths needs to be updated
return remote_dir.partition('/')[0], remote_dir.partition('/')[2]
class Downloader(Base):
""" Downloader for OpenStack swift object storages
"""
def download(self, remote_path: os.PathLike, local_path: os.PathLike) -> os.PathLike:
if self.container:
# container was specified, use it
container = self.container
else:
# container needs to be extracted from path
# paths needs to be updated
container = remote_path.partition('/')[0]
remote_path = remote_path.partition('/')[2]
container, remote_path = self.validate_container(remote_path)
target_filename = os.path.join(local_path, os.path.basename(remote_path))
with self.get_service() as swift:
results = swift.download(
......@@ -74,6 +75,7 @@ class Uploader(Base):
"""
def upload(self, local_path: Union[os.PathLike, List[os.PathLike]], remote_dir: os.PathLike) -> List[os.PathLike]:
paths = local_path if isinstance(local_path, List) else [local_path]
container, remote_dir = self.validate_container(remote_dir)
remote_paths = [
os.path.join(
remote_dir,
......@@ -99,7 +101,6 @@ class Uploader(Base):
with self.get_service() as swift:
# use container or first part of path
container = self.container or remote_dir.partition('/')[0]
results = swift.upload(container=container, objects=objects, options=options)
for result in results:
......@@ -123,8 +124,7 @@ class Uploader(Base):
def product_exists(self, remote_dir: os.PathLike) -> bool:
with self.get_service() as swift:
# use container or first part of path
container = self.container or remote_dir.partition('/')[0]
container, remote_dir = self.validate_container(remote_dir)
list_parts_gen = swift.list(
container=container, options={"prefix": remote_dir},
)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment