EOX GitLab Instance

Commit dfa3cdd8 authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Fixing handling of absolute paths in S3 source

parent 51425b69
Pipeline #19463 failed with stage
in 40 seconds
import logging
from typing import TYPE_CHECKING, IO, AnyStr, List
from typing import TYPE_CHECKING, IO, AnyStr, List, Tuple
from functools import cached_property
from urllib.parse import urlparse
......@@ -58,14 +58,19 @@ class S3Source(Source):
)
return client
def get_bucket_and_key(self, path):
def get_bucket_and_key(self, path: str) -> Tuple[str, str]:
parsed = urlparse(path)
if parsed.scheme and parsed.scheme.lower() != "s3":
raise ValueError(f"Invalid S3 URL {path}")
if parsed.netloc:
return (parsed.netloc, parsed.path)
path = parsed.path
if path.startswith("/"):
path = path[1:]
return (parsed.netloc, path)
if path.startswith("/"):
path = path[1:]
return (self.bucket, path)
def listdir(self, path) -> List[str]:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment