diff --git a/preprocessor/preprocessor/transfer/swift.py b/preprocessor/preprocessor/transfer/swift.py index ad57dabcfbfb73b99ab9af169644d05fadea8b86..a58f7fb146530860a1b7af61961b38ea6b6849dc 100644 --- a/preprocessor/preprocessor/transfer/swift.py +++ b/preprocessor/preprocessor/transfer/swift.py @@ -39,19 +39,20 @@ class Base: "os_user_domain_name": self.user_domain_name, }) + def validate_container(self, remote_dir): + if self.container: + # container was specified, use it + return self.container, remote_dir + # container needs to be extracted from path + # paths needs to be updated + return remote_dir.partition('/')[0], remote_dir.partition('/')[2] + class Downloader(Base): """ Downloader for OpenStack swift object storages """ def download(self, remote_path: os.PathLike, local_path: os.PathLike) -> os.PathLike: - if self.container: - # container was specified, use it - container = self.container - else: - # container needs to be extracted from path - # paths needs to be updated - container = remote_path.partition('/')[0] - remote_path = remote_path.partition('/')[2] + container, remote_path = self.validate_container(remote_path) target_filename = os.path.join(local_path, os.path.basename(remote_path)) with self.get_service() as swift: results = swift.download( @@ -74,6 +75,7 @@ class Uploader(Base): """ def upload(self, local_path: Union[os.PathLike, List[os.PathLike]], remote_dir: os.PathLike) -> List[os.PathLike]: paths = local_path if isinstance(local_path, List) else [local_path] + container, remote_dir = self.validate_container(remote_dir) remote_paths = [ os.path.join( remote_dir, @@ -99,7 +101,6 @@ class Uploader(Base): with self.get_service() as swift: # use container or first part of path - container = self.container or remote_dir.partition('/')[0] results = swift.upload(container=container, objects=objects, options=options) for result in results: @@ -123,8 +124,7 @@ class Uploader(Base): def product_exists(self, remote_dir: os.PathLike) -> bool: with self.get_service() as swift: - # use container or first part of path - container = self.container or remote_dir.partition('/')[0] + container, remote_dir = self.validate_container(remote_dir) list_parts_gen = swift.list( container=container, options={"prefix": remote_dir}, )