EOX GitLab Instance

Commit 103012c9 authored by Fabian Schindler's avatar Fabian Schindler
Browse files

Fixing ABCs for sources

Adding filescheme setup
Adding config example
parent 6dcd0032
......@@ -17,3 +17,17 @@ harvesters:
- - P5D
- !now
queue: register # register, ingest, delete, update, preprocess
- name: MyS3STACCatalogHarvester
type: STACCatalog
source:
type: S3
bucket: mybucket
secret_access_key: xxx
access_key_id: xxx
endpoint_url: myendpoint.storage.com
validate_bucket_name: False
region_name: RegionA
public: False
# path is not explicitly specified, but must be passed as argument
# path:
from typing import Optional
from ..abc import FileScheme
from ..source import get_source
from .filematcher import FileMatcherScheme
from .stac_catalog import STACCatalogScheme
SCHEME_MAP = {
"FileMatcher": FileMatcherScheme,
"STACCatalog": STACCatalogScheme
}
def get_filescheme(filescheme_cfg: dict) -> Optional[FileScheme]:
cls = SCHEME_MAP.get(filescheme_cfg["type"])
if not cls:
return None
parameters = filescheme_cfg["parameters"]
filescheme = cls(get_source(filescheme_cfg["source"]), **parameters)
return filescheme
......@@ -6,7 +6,7 @@ from redis import Redis
from .abc import Resource
from .endpoint import get_endpoint
from .source import get_source
from .filescheme import get_filescheme
from .exceptions import HarvestError
from .utils import cql_filter
......@@ -30,7 +30,7 @@ def init_resource(harvest_config: dict) -> Resource:
if endpoint := get_endpoint(config):
return endpoint
if source := get_source(config):
if source := get_filescheme(config):
return source
raise HarvestError(f"Resource type {config['type']} not found")
......
......@@ -13,23 +13,16 @@ SOURCE_MAP = {
def get_source(source_cfg: dict) -> Optional[Source]:
cls = SOURCE_MAP.get(source_cfg["type"])
cls = SOURCE_MAP.get(source_cfg.pop("type"))
if not cls:
return None
parameters = source_cfg["parameters"]
source = cls(parameters)
source = cls(**source_cfg)
return source
__all__ = [
"FTPSource",
"S3Source",
"S3CatalogSource",
"STACCatalogSource",
"SwiftSource",
"Source",
"get_source",
]
from typing import Iterator
from ._source import Source
from ..abc import Source
class FTPSource(Source):
......
......@@ -26,7 +26,6 @@ class S3Source(Source):
secret_access_key: str = None,
access_key_id: str = None,
endpoint_url: str = "",
strip_bucket: bool = True,
validate_bucket_name: bool = True,
region_name: str = None,
public: bool = False,
......@@ -35,7 +34,6 @@ class S3Source(Source):
self.secret_access_key = secret_access_key
self.access_key_id = access_key_id
self.endpoint_url = endpoint_url
self.strip_bucket = strip_bucket
self.region_name = region_name
self.public = public
self.validate_bucket_name = validate_bucket_name
......
from typing import Iterator
from ._source import Source
from ..abc import Source
class SwiftSource(Source):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment