Fixing ABCs for sources

Adding filescheme setup
Adding config example

config-sample.yaml

@@ -17,3 +17,17 @@ harvesters:
               - - P5D
                 - !now
     queue: register # register, ingest, delete, update, preprocess
+
+  - name: MyS3STACCatalogHarvester
+    type: STACCatalog
+    source:
+      type: S3
+      bucket: mybucket
+      secret_access_key: xxx
+      access_key_id: xxx
+      endpoint_url: myendpoint.storage.com
+      validate_bucket_name: False
+      region_name: RegionA
+      public: False
+    # path is not explicitly specified, but must be passed as argument
+    # path:

harvester/filescheme/__init__.py

+from typing import Optional
+
+from ..abc import FileScheme
+from ..source import get_source
+from .filematcher import FileMatcherScheme
+from .stac_catalog import STACCatalogScheme
+
+
+SCHEME_MAP = {
+    "FileMatcher": FileMatcherScheme,
+    "STACCatalog": STACCatalogScheme
+}
+
+
+def get_filescheme(filescheme_cfg: dict) -> Optional[FileScheme]:
+    cls = SCHEME_MAP.get(filescheme_cfg["type"])
+    if not cls:
+        return None
+
+    parameters = filescheme_cfg["parameters"]
+    filescheme = cls(get_source(filescheme_cfg["source"]), **parameters)
+
+    return filescheme

harvester/harvester.py

@@ -6,7 +6,7 @@ from redis import Redis
 
 from .abc import Resource
 from .endpoint import get_endpoint
-from .source import get_source
+from .filescheme import get_filescheme
 from .exceptions import HarvestError
 from .utils import cql_filter
 
@@ -30,7 +30,7 @@ def init_resource(harvest_config: dict) -> Resource:
     if endpoint := get_endpoint(config):
         return endpoint
 
-    if source := get_source(config):
+    if source := get_filescheme(config):
         return source
 
     raise HarvestError(f"Resource type {config['type']} not found")

harvester/source/__init__.py

@@ -13,23 +13,16 @@ SOURCE_MAP = {
 
 
 def get_source(source_cfg: dict) -> Optional[Source]:
-
-    cls = SOURCE_MAP.get(source_cfg["type"])
+    cls = SOURCE_MAP.get(source_cfg.pop("type"))
     if not cls:
         return None
-    parameters = source_cfg["parameters"]
-
-    source = cls(parameters)
-
+    source = cls(**source_cfg)
     return source
 
 
 __all__ = [
     "FTPSource",
     "S3Source",
-    "S3CatalogSource",
-    "STACCatalogSource",
     "SwiftSource",
-    "Source",
     "get_source",
 ]

harvester/source/ftp.py

 from typing import Iterator
-from ._source import Source
+from ..abc import Source
 
 
 class FTPSource(Source):

harvester/source/s3.py

@@ -26,7 +26,6 @@ class S3Source(Source):
         secret_access_key: str = None,
         access_key_id: str = None,
         endpoint_url: str = "",
-        strip_bucket: bool = True,
         validate_bucket_name: bool = True,
         region_name: str = None,
         public: bool = False,
@@ -35,7 +34,6 @@ class S3Source(Source):
         self.secret_access_key = secret_access_key
         self.access_key_id = access_key_id
         self.endpoint_url = endpoint_url
-        self.strip_bucket = strip_bucket
         self.region_name = region_name
         self.public = public
         self.validate_bucket_name = validate_bucket_name

harvester/source/swift.py

 from typing import Iterator
-from ._source import Source
+from ..abc import Source
 
 
 class SwiftSource(Source):