EOX GitLab Instance

Commit dfa3cdd8 authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Fixing handling of absolute paths in S3 source

parent 51425b69
Pipeline #19463 failed with stage
in 40 seconds
import logging import logging
from typing import TYPE_CHECKING, IO, AnyStr, List from typing import TYPE_CHECKING, IO, AnyStr, List, Tuple
from functools import cached_property from functools import cached_property
from urllib.parse import urlparse from urllib.parse import urlparse
...@@ -58,14 +58,19 @@ class S3Source(Source): ...@@ -58,14 +58,19 @@ class S3Source(Source):
) )
return client return client
def get_bucket_and_key(self, path): def get_bucket_and_key(self, path: str) -> Tuple[str, str]:
parsed = urlparse(path) parsed = urlparse(path)
if parsed.scheme and parsed.scheme.lower() != "s3": if parsed.scheme and parsed.scheme.lower() != "s3":
raise ValueError(f"Invalid S3 URL {path}") raise ValueError(f"Invalid S3 URL {path}")
if parsed.netloc: if parsed.netloc:
return (parsed.netloc, parsed.path) path = parsed.path
if path.startswith("/"):
path = path[1:]
return (parsed.netloc, path)
if path.startswith("/"):
path = path[1:]
return (self.bucket, path) return (self.bucket, path)
def listdir(self, path) -> List[str]: def listdir(self, path) -> List[str]:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment