From bdedd34ccdb4a599e7a8597c075e7b8aad4bd43e Mon Sep 17 00:00:00 2001
From: Fabian Schindler <fabian.schindler.strauss@gmail.com>
Date: Fri, 1 Apr 2022 09:43:07 +0200
Subject: [PATCH] Adding HTTP source
---
registrar/backend/eoxserver.py | 11 ++++++++++-
registrar/source.py | 23 ++++++++++++++++++++++-
2 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/registrar/backend/eoxserver.py b/registrar/backend/eoxserver.py
index 08c652f..b011f74 100644
--- a/registrar/backend/eoxserver.py
+++ b/registrar/backend/eoxserver.py
@@ -22,7 +22,7 @@ if TYPE_CHECKING:
from pystac import Item
from ..exceptions import RegistrationError
-from ..source import Source, LocalSource, S3Source, SwiftSource
+from ..source import HTTPSource, Source, LocalSource, S3Source, SwiftSource
from .abc import ItemBackend
@@ -206,6 +206,15 @@ class EOxServerBackend(ItemBackend):
)
storage_name = storage.name
+ elif isinstance(source, HTTPSource):
+ storage, created_storage = backends.Storage.objects.get_or_create(
+ name=source.name,
+ url=source.endpoint_url,
+ storage_type="http",
+ # TODO: maybe add storage auth at some point
+ )
+ storage_name = storage.name
+
if created_storage_auth:
logger.info("Created storage auth", source=source.name)
if created_storage:
diff --git a/registrar/source.py b/registrar/source.py
index c4a68b9..2ea8ad1 100644
--- a/registrar/source.py
+++ b/registrar/source.py
@@ -11,10 +11,12 @@ from os.path import normpath, join, isabs
import shutil
from glob import glob
from fnmatch import fnmatch
+
+import requests
import structlog
from typing import TYPE_CHECKING, Optional
from abc import ABC, abstractmethod
-from urllib.parse import urlparse
+from urllib.parse import urljoin, urlparse
import boto3
import boto3.session
@@ -369,10 +371,29 @@ class LocalSource(Source):
return {}, self._join_path(path)
+class HTTPSource(Source):
+ def get_container_and_path(self, path: str):
+ return (self.endpoint_url, path)
+
+ def list_files(self, path: str, glob_pattern: list = None):
+ raise NotImplementedError()
+
+ def get_file(self, path: str, target_path: str):
+ url = urljoin(self.endpoint_url, path)
+ response = requests.get(url, allow_redirects=True)
+ with open(target_path, "w") as f:
+ f.write(response.content)
+
+ @abstractmethod
+ def get_vsi_env_and_path(self, path: str):
+ return {}, f"/vsicurl/{urljoin(self.endpoint_url, path)}"
+
+
SOURCE_TYPES = {
"swift": SwiftSource,
"s3": S3Source,
"local": LocalSource,
+ "http": HTTPSource,
}
--
GitLab