Merge branch 'postprocess' into 'main'

Postprocessing

See merge request !6

harvester/harvester.py

@@ -10,7 +10,7 @@ from .endpoint import get_endpoint
 from .source import get_source
 from .exceptions import HarvestError
 from .utils import cql_filter
-
+from .postprocess import get_postprocessor
 logger = logging.getLogger(__name__)
 
 
@@ -52,6 +52,13 @@ def main(config: dict, value: str, client: Redis):
     # Perform harvest
     result = resource.harvest()
 
+    if "postprocess" in harvest_config:
+        postprocessor = get_postprocessor(harvest_config["postprocess"])
+        result = (
+            postprocessor.postprocess(item)
+            for item in result
+        )
+
     # Filter data
     result = cql_filter(harvest_config["filter"], result)
 

harvester/postprocess.py

+from abc import ABC, abstractmethod
+from typing import Dict, Type
+
+
+class Postprocessor(ABC):
+    def __init__(self, **kwargs):
+        ...
+
+    @abstractmethod
+    def postprocess(self, item: dict) -> dict:
+        pass
+
+
+POSTPROCESSORS: Dict[str, Type[Postprocessor]] = {
+
+}
+
+
+def get_postprocessor(config: dict) -> Postprocessor:
+    cls = POSTPROCESSORS[config.pop("type")]
+    return cls(**config)