diff --git a/registrar/backend/eoxserver.py b/registrar/backend/eoxserver.py index 08c652f90561e558037a519605bbde6c4b97c330..b011f74b2d48a26afe78efcf2c610bedc6085be3 100644 --- a/registrar/backend/eoxserver.py +++ b/registrar/backend/eoxserver.py @@ -22,7 +22,7 @@ if TYPE_CHECKING: from pystac import Item from ..exceptions import RegistrationError -from ..source import Source, LocalSource, S3Source, SwiftSource +from ..source import HTTPSource, Source, LocalSource, S3Source, SwiftSource from .abc import ItemBackend @@ -206,6 +206,15 @@ class EOxServerBackend(ItemBackend): ) storage_name = storage.name + elif isinstance(source, HTTPSource): + storage, created_storage = backends.Storage.objects.get_or_create( + name=source.name, + url=source.endpoint_url, + storage_type="http", + # TODO: maybe add storage auth at some point + ) + storage_name = storage.name + if created_storage_auth: logger.info("Created storage auth", source=source.name) if created_storage: diff --git a/registrar/source.py b/registrar/source.py index c4a68b9bd7c995a6ea9298735b701438c1113b36..2ea8ad1040e333844d60272c729c1ac2575f28c7 100644 --- a/registrar/source.py +++ b/registrar/source.py @@ -11,10 +11,12 @@ from os.path import normpath, join, isabs import shutil from glob import glob from fnmatch import fnmatch + +import requests import structlog from typing import TYPE_CHECKING, Optional from abc import ABC, abstractmethod -from urllib.parse import urlparse +from urllib.parse import urljoin, urlparse import boto3 import boto3.session @@ -369,10 +371,29 @@ class LocalSource(Source): return {}, self._join_path(path) +class HTTPSource(Source): + def get_container_and_path(self, path: str): + return (self.endpoint_url, path) + + def list_files(self, path: str, glob_pattern: list = None): + raise NotImplementedError() + + def get_file(self, path: str, target_path: str): + url = urljoin(self.endpoint_url, path) + response = requests.get(url, allow_redirects=True) + with open(target_path, "w") as f: + f.write(response.content) + + @abstractmethod + def get_vsi_env_and_path(self, path: str): + return {}, f"/vsicurl/{urljoin(self.endpoint_url, path)}" + + SOURCE_TYPES = { "swift": SwiftSource, "s3": S3Source, "local": LocalSource, + "http": HTTPSource, }