EOX GitLab Instance

Skip to content
Snippets Groups Projects
Commit 0c80f355 authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Adding way to handle fully qualified S3 URLs

parent ea60f9de
No related branches found
Tags release-2.0.13
No related merge requests found
......@@ -14,6 +14,7 @@ from fnmatch import fnmatch
import logging
from typing import TYPE_CHECKING, Optional
from abc import ABC, abstractmethod
from urllib.parse import urlparse
import boto3
import boto3.session
......@@ -253,6 +254,14 @@ class S3Source(Source):
)
def get_container_and_path(self, path: str):
# try to see if we have a fully qualified S3 URL, in which case we will
# return bucket/path from there
parsed = urlparse(path)
if parsed.scheme:
if parsed.scheme.lower() != "s3":
raise ValueError("invalid S3 URL {path}")
return (parsed.netloc, parsed.path[1:])
bucket = self.bucket_name
if bucket is None:
parts = (path[1:] if path.startswith("/") else path).split("/")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment